Preface: machine learning engineers spend roughly half their time on data cleaning, feature selection, dimensionality reduction, and other data processing. Below, using an email filtering system as an example, we introduce some very important work before building a machine learning model.

Data Collection

Different projects have different data sources, as discussed earlier.

Inspecting the Data

The data for training this model consists of more than sixty thousand emails and their labels, as shown below:

Emails

Labels

From the data we can determine the following:

Task

• Supervised or unsupervised learning? Binary or multi-class classification? Text or structured data classification? Short or long text classification? Answer: labels are available — supervised learning, binary classification, long-text classification.

Data

• How are samples defined? What data serves as features? How should we split training and test sets? Answer: features can include sender address, recipient address, send time, email body, and email length. How do we select appropriate features from the above? Answer: through statistical computation. Choose an appropriate model; optimize the model for the specific task; tune the model; ensemble multiple models.

Data Preprocessing

• Extract the above features into a CSV file.

1. Convert labels to numbers. Code:

import sys
import os
import time
'''
把六万条数据，写到一行上，制作标签，标签已经给你标注好
'''
#1制作标签字典
def label_dict(label_path):
    type_dict = {"spam":"1","ham":"0"}
    content = open(label_path)
    index_dict = {}
    #用try防止出错发生
    try:
        for line in content:
            arr = line.split(" ")
            if len(arr)==2:
                key,value=arr
                value=value.replace("../data",'').replace("\n",'')
                index_dict[value]=type_dict[key.lower()]
    finally:
        content.close()
    return index_dict
a = label_dict("./full/index")
print(a)

Output:

'/028/239': '0', '/028/240': '0', '/028/241': '1', '/028/242': '1', '/028/243': '1', '/028/244': '1', '/028/245': '1', '/028/2

2. Extract features — first define feature extraction for a single file:

def feature_dict(email_path):
    email_content = open(email_path,'r',encoding="gb2312",errors="ignore")
    content_dict={}
    try:
        is_content = False
        for line in email_content:
            line = line.strip()#去除首尾空格字符
            if line.startswith("From:"):
                content_dict["from"] = line[5:]
            elif line.startswith("To"):
                content_dict["to"]=line[3:]
            elif line.startswith("Date"):
                content_dict["date"]=line[5:]
            elif not line:
                is_content=True
            if is_content:
                if "content" in content_dict:
                    content_dict['content'] += line
                else:
                    content_dict['content'] = line
        pass
    finally:
        email_content.close()
    return content_dict

Output:

{'from': ' "yan"', 'to': ' lu@ccert.edu.cn', 'date': ' Tue, 30 Aug 2005 10:08:15 +0800', 'content': '非财务纠淼牟莆窆芾-（沙盘模拟

3. Convert the dictionary above to text:

def dict_to_text(email_path):
    content_dict=feature_dict(email_path)
    # 进行处理
    result_str = content_dict.get('from', 'unkown').replace(',', '').strip() + ","
    result_str += content_dict.get('to', 'unknown').replace(',', '').strip() + ","
    result_str += content_dict.get('date', 'unknown').replace(',', '').strip() + ","
    result_str += content_dict.get('content', 'unknown').replace(',', ' ').strip()
    return result_str

Output:

"yan",lu@ccert.edu.cn,Tue 30 Aug 2005 10:08:15 +0800,非财务纠淼牟莆窆芾-（沙盘模拟）------如何运用财务岳硖岣吖芾砑

4. Extract the above features and write them to a file — two nested for loops:

start = time.time()
index_dict = label_dict("./full/index")
list0 = os.listdir('./data')  # 文件夹的名称

for l1 in list0:  # 开始把N个文件夹中的file写入N*n个wiriter
    l1_path = './data/' + l1
    print('开始处理文件夹' + l1_path)
    list1 = os.listdir(l1_path)

    write_file_path = './process/process01_' + l1

    with open(write_file_path, "w", encoding='utf-8') as writer:
        for l2 in list1:
            l2_path = l1_path + "/" + l2  # 得到要处理文件的具体路径

            index_key = "/" + l1 + "/" + l2

            if index_key in index_dict:
                content_str = dict_to_text(l2_path)
                content_str += "," + index_dict[index_key] + "\n"
                writer.writelines(content_str)

with open('./result_process01', "w", encoding='utf-8') as writer:
    for l1 in list0:
        file_path = './process/process01_' + l1
        print("开始合并文件：" + file_path)

        with open(file_path, encoding='utf-8') as file:
            for line in file:
                writer.writelines(line)

end = time.time()

print('数据处理总共耗时%.2f' % (end - start))

Result:

New file

Data Analysis

Examine the correlation between each feature attribute and the label value.

1. Check the effect of sender and recipient addresses on the label:

df = pd.read_csv('./result_process01', sep = ',', header = None, names= ['from','to', 'date', 'content','label'])
def 获取邮件收发地址(strl):#发送接收地址提取
    it = re.findall(r"@([A-Za-z0-9]*\.[A-Za-z0-9\.]+)", str(strl))#正则匹配
    result = ''
    if len(it)>0:
        result = it[0]
    else:
        result = 'unknown'
    return result

df['from_address'] = pd.Series(map(lambda str : 获取邮件收发地址(str), df['from']))#map映射并添加
df['to_address'] = pd.Series(map(lambda str: 获取邮件收发地址(str), df['to']))
#开始分析：多少种地址，每种多少个
print(df['from_address'].unique().shape)
print(df['from_address'].value_counts().head(5))
from_address_df = df.from_address.value_counts().to_frame()#转为结构化的输出,输出带索引
print(from_address_df.head(5))

Result:

(3567,)
163.com                  7500
mail.tsinghua.edu.cn     6498
126.com                  5822
tom.com                  4075
mails.tsinghua.edu.cn    3205

We can see that address has little effect on whether an email is spam. Time also has little effect.

2. Segment the content:

print('='*30 + '现在开始分词，请耐心等待5分钟。。。' + '='*20)
df['content'] = df['content'].astype('str')#astype类型转换,转为str
df['jieba_cut_content'] = list(map(lambda st: "  ".join(jieba.cut(st)), df['content']))
print(df["jieba_cut_content"].head(4))

3. Check whether email length affects whether an email is spam:

def 邮件长度统计(lg):
    if lg

![image](/writing/ml/case-study-feature-engineering/img-04.png)

Email length clearly has some effect. The fitted function:

![image](/writing/ml/case-study-feature-engineering/img-05.png)

def process_content_sema(x): if x > 10000: return 0.5 / np.exp(np.log10(x) - np.log10(500)) + np.log(abs(x - 500) + 1) - np.log(abs(x - 10000)) + 1 else: return 0.5 / np.exp(np.log10(x) - np.log10(500)) + np.log(abs(x - 500) + 1)

**4. Feature extraction — remove useless features and save the useful ones:**

df[‘content_length_sema’] = list(map(lambda st: process_content_sema(st), df[‘content_length’]))

print(df.head(10))

sys.exit(0)

print(df.dtypes) #可以查看每一列的数据类型，也可以查看每一列的名称

df.drop([‘from’, ‘to’, ‘date’, ‘from_address’, ‘to_address’,
‘date_week’,‘date_hour’, ‘date_time_quantum’, ‘content’,
‘content_length’, ‘content_length_type’], 1, inplace=True) print(df.info()) print(df.head(10))

df.to_csv(’./result_process02’, encoding=‘utf-8’, index = False) df.to_csv(’./result_process02.csv’, encoding=‘utf-8’, index = False)

Result:

![image](/writing/ml/case-study-feature-engineering/img-06.png)

### Model Training

We choose the Naive Bayes algorithm because it is fast and effective. Recall is used to evaluate the model.

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer#CountVectorizer把词进行可视化 from sklearn.decomposition import TruncatedSVD from sklearn.naive_bayes import BernoulliNB from sklearn.model_selection import train_test_split from sklearn.metrics import f1_score, precision_score, recall_score

mpl.rcParams[‘font.sans-serif’] = [u’simHei’]

mpl.rcParams[‘axes.unicode_minus’] = False

df = pd.read_csv(’./result_process02.csv’, sep =’,‘)

print(df.head(5))

df.dropna(axis = 0, how =‘any’, inplace = True) #按行删除Nan 确保数据安全

print(df.head(5))

print(df.info())

x_train, x_test, y_train, y_test = train_test_split(df[[‘has_date’,‘jieba_cut_content’]],
df[‘label’],test_size = 0.2, random_state = 0)

print(“训练数据集大小：%d” % x_train.shape[0])

print(“测试集数据大小：%d” % x_test.shape[0])

print(x_train.head(10))

print(x_test.head(10)) #注意前面索引

#================================================================================================ print(’=‘*30 + ‘开始计算tf—idf权重’ + ’=‘*30) transformer = TfidfVectorizer(norm = ‘l2’, use_idf = True)#逆向文件频率 svd = TruncatedSVD(n_components=20) jieba_cut_content = list(x_train[‘jieba_cut_content’].astype(‘str’)) transformer_model = transformer.fit(jieba_cut_content) df1 = transformer_model.transform(jieba_cut_content)

print(df1)

print(df1.shape)

print(’=‘*30 + ‘开始SVD降维计算’ + ’=‘*30) svd_model = svd.fit(df1) df2 = svd_model.transform(df1) data = pd.DataFrame(df2)

print(data.head(10))

print(data.info())

print(’=‘*30 + ‘重新构建矩阵开始’ + ’=‘*30) data[‘has_date’] = list(x_train[‘has_date’])

data[‘content_length_sema’] = list(x_train[‘content_length_sema’])

print(data.head(10))

print(data.info())

print(’=‘*30 + ‘构建伯努利贝叶斯模型’ + ’=‘*30) nb = BernoulliNB(alpha = 1.0, binarize = 0.0005)#二值转换阈值 model = nb.fit(data, y_train) #================================================================================ print(’=‘*30 + ‘构建测试集’ + ’=‘*30) jieba_cut_content_test = list(x_test[‘jieba_cut_content’].astype(‘str’)) data_test = pd.DataFrame(svd_model.transform(transformer_model.transform(jieba_cut_content_test))) data_test[‘has_date’] = list(x_test[‘has_date’])

data_test[‘content_length_sema’] = list(x_test[‘content_length_sema’])

print(data_test.head(10))

print(data_test.info())

#开始预测 print(’=‘*30 + ‘开始预测测试集’ + ’=‘*30) y_predict = model.predict(data_test)

precision = precision_score(y_test, y_predict) recall = recall_score(y_test, y_predict) f1mean = f1_score(y_test, y_predict)

print(‘精确率为：%0.5f’ % precision) print(‘召回率：%0.5f’ % recall) print(‘F1均值为：%0.5f’ % f1mean)

Results: precision: 0.94549; recall: 0.98925; F1 mean: 0.96688.

Full code and documentation: https://github.com/dctongsheng/Spam-filtering-projects001