如何用源码搭建网站,新网站一直不被收录,东莞手机网站站定制开发,网站五合一建设一、任务背景 本次python实战#xff0c;我们使用来自Kaggle的数据集《Chinese MNIST》进行CNN分类建模#xff0c;不同于经典的MNIST数据集#xff0c;我们这次使用的数据集是汉字手写体数字。除了常规的汉字“零”到“九”之外还多了“十”、“百”、“千”、“万”、“亿…一、任务背景 本次python实战我们使用来自Kaggle的数据集《Chinese MNIST》进行CNN分类建模不同于经典的MNIST数据集我们这次使用的数据集是汉字手写体数字。除了常规的汉字“零”到“九”之外还多了“十”、“百”、“千”、“万”、“亿”共15种汉字数字。 二、python建模
1、数据读取 首先读取jpg数据文件可以看到总共有15000张图像数据。
import pandas as pd
import ospath /kaggle/input/chinese-mnist/data/data/
files os.listdir(path)
print(数据总量, len(files)) 我们也可以打印一张图片出来看看。
import matplotlib.pyplot as plt
import matplotlib.image as mpimg# 定义图片路径
image_path pathfiles[3]# 加载图片
image mpimg.imread(image_path)# 绘制图片
plt.figure(figsize(3, 3))
plt.imshow(image)
plt.axis(off) # 关闭坐标轴
plt.show() 2、数据集构建 加载必要的库以便后续使用再定义一些超参数。
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchvision import transforms
from PIL import Image
from torch.utils.data import Dataset, DataLoader, random_split
from sklearn.metrics import precision_score, recall_score, f1_score# 超参数
batch_size 64
learning_rate 0.01
num_epochs 5# 数据预处理
transform transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))
]) 这里我们看一看数据集介绍就会知道图片名称及其含义需要从chinese_mnist.csv文件中根据图片名称中的几个数字来确定图片对应的标签。 # 获取所有图片文件的路径
all_images [os.path.join(path, img) for img in os.listdir(path) if img.endswith(.jpg)]# 读取索引-标签对应关系csv文件并将suite_id, sample_id, code设置为索引列便于查找
index_df pd.read_csv(/kaggle/input/chinese-mnist/chinese_mnist.csv)
index_df.set_index([suite_id, sample_id, code], inplaceTrue)# 定义函数根据各索引取值定位图片对应的数值标签value
def get_label_from_index(filename, index_df):suite_id, sample_id, code map(int, filename.split(.)[0].split(_)[1:])return index_df.loc[(suite_id, sample_id, code), value]# 构建value值对应的标签序号用于模型训练
label_dic {0:0, 1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8, 9:9, 10:10, 100:11, 1000:12, 10000:13, 100000000:14}
# 获取所有图片的标签并转化为标签序号
all_labels [get_label_from_index(os.path.basename(img), index_df) for img in all_images]
all_labels [label_dic[li] for li in all_labels]# 将图片路径和标签分成训练集和测试集
train_images, test_images, train_labels, test_labels train_test_split(all_images, all_labels, test_size0.2, random_state2024) 下面定义数据集类并完成数据的加载。
# 自定义数据集类
class CustomDataset(Dataset):def __init__(self, image_paths, labels, transformNone):self.image_paths image_pathsself.labels labelsself.transform transformdef __len__(self):return len(self.image_paths)def __getitem__(self, idx):image Image.open(self.image_paths[idx]).convert(L) # 转换为灰度图像label self.labels[idx]if self.transform:image self.transform(image)return image, label# 创建训练集和测试集数据集
train_dataset CustomDataset(train_images, train_labels, transformtransform)
test_dataset CustomDataset(test_images, test_labels, transformtransform)# 创建数据加载器
train_loader DataLoader(datasettrain_dataset, batch_size64, shuffleTrue)
test_loader DataLoader(datasettest_dataset, batch_size64, shuffleFalse)# 打印一些信息
print(f训练集样本数: {len(train_dataset)})
print(f测试集样本数: {len(test_dataset)})
3、模型构建 我们构建一个包含两层卷积层和池化层的CNN并且在池化层中使用最大池化的方式。
# 定义CNN模型
class CNN(nn.Module):def __init__(self):super(CNN, self).__init__()self.conv1 nn.Conv2d(1, 32, kernel_size3, padding1)self.conv2 nn.Conv2d(32, 64, kernel_size3, padding1)self.pool nn.MaxPool2d(kernel_size2, stride2, padding0)self.fc1 nn.Linear(64 * 16 * 16, 128)self.fc2 nn.Linear(128, 15)def forward(self, x):x self.pool(F.relu(self.conv1(x)))x self.pool(F.relu(self.conv2(x)))x x.view(-1, 64 * 16 * 16)x F.relu(self.fc1(x))x self.fc2(x)return x
4、模型实例化及训练 下面我们对模型进行实例化并定义criterion和optimizer。
# 初始化模型、损失函数和优化器
model CNN()
criterion nn.CrossEntropyLoss()
optimizer optim.SGD(model.parameters(), lrlearning_rate, momentum0.9) 定义训练的代码并调用代码训练模型。
from tqdm import tqdm
# 训练模型
def train(model, train_loader, criterion, optimizer, epochs):model.train()running_loss 0.0for epoch in range(epochs):for data, target in tqdm(train_loader):optimizer.zero_grad()output model(data)loss criterion(output, target)loss.backward()optimizer.step()running_loss loss.item()print(fEpoch [{epoch 1}], Loss: {running_loss / len(train_loader):.4f})running_loss 0.0train(model, train_loader, criterion, optimizer, num_epochs) 5、测试模型 定义模型测试代码调用代码看指标可知我们所构建的CNN模型表现还不错。
# 测试模型
def test(model, test_loader, criterion):model.eval()test_loss 0correct 0all_preds []all_targets []with torch.no_grad():for data, target in test_loader:output model(data)test_loss criterion(output, target).item()pred output.argmax(dim1, keepdimTrue)correct pred.eq(target.view_as(pred)).sum().item()all_preds.extend(pred.cpu().numpy())all_targets.extend(target.cpu().numpy())test_loss / len(test_loader.dataset)accuracy 100. * correct / len(test_loader.dataset)precision precision_score(all_targets, all_preds, averagemacro)recall recall_score(all_targets, all_preds, averagemacro)f1 f1_score(all_targets, all_preds, averagemacro)print(fTest Loss: {test_loss:.4f}, Accuracy: {accuracy:.2f}%, Precision: {precision:.4f}, Recall: {recall:.4f}, F1 Score: {f1:.4f})test(model, test_loader, criterion) 三、完整代码
import pandas as pd
import os
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchvision import transforms
from PIL import Image
from tqdm import tqdm
from torch.utils.data import Dataset, DataLoader, random_split
from sklearn.metrics import precision_score, recall_score, f1_scorepath /kaggle/input/chinese-mnist/data/data/
files os.listdir(path)
print(数据总量, len(files))# 超参数
batch_size 64
learning_rate 0.01
num_epochs 5# 数据预处理
transform transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))
])# 获取所有图片文件的路径
all_images [os.path.join(path, img) for img in os.listdir(path) if img.endswith(.jpg)]# 读取索引-标签对应关系csv文件并将suite_id, sample_id, code设置为索引列便于查找
index_df pd.read_csv(/kaggle/input/chinese-mnist/chinese_mnist.csv)
index_df.set_index([suite_id, sample_id, code], inplaceTrue)# 定义函数根据各索引取值定位图片对应的数值标签value
def get_label_from_index(filename, index_df):suite_id, sample_id, code map(int, filename.split(.)[0].split(_)[1:])return index_df.loc[(suite_id, sample_id, code), value]# 构建value值对应的标签序号用于模型训练
label_dic {0:0, 1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8, 9:9, 10:10, 100:11, 1000:12, 10000:13, 100000000:14}# 获取所有图片的标签并转化为标签序号
all_labels [get_label_from_index(os.path.basename(img), index_df) for img in all_images]
all_labels [label_dic[li] for li in all_labels]# 将图片路径和标签分成训练集和测试集
train_images, test_images, train_labels, test_labels train_test_split(all_images, all_labels, test_size0.2, random_state2024)# 自定义数据集类
class CustomDataset(Dataset):def __init__(self, image_paths, labels, transformNone):self.image_paths image_pathsself.labels labelsself.transform transformdef __len__(self):return len(self.image_paths)def __getitem__(self, idx):image Image.open(self.image_paths[idx]).convert(L) # 转换为灰度图像label self.labels[idx]if self.transform:image self.transform(image)return image, label# 创建训练集和测试集数据集
train_dataset CustomDataset(train_images, train_labels, transformtransform)
test_dataset CustomDataset(test_images, test_labels, transformtransform)# 创建数据加载器
train_loader DataLoader(datasettrain_dataset, batch_size64, shuffleTrue)
test_loader DataLoader(datasettest_dataset, batch_size64, shuffleFalse)# 打印信息
print(f训练集样本数: {len(train_dataset)})
print(f测试集样本数: {len(test_dataset)})# 定义CNN模型
class CNN(nn.Module):def __init__(self):super(CNN, self).__init__()self.conv1 nn.Conv2d(1, 32, kernel_size3, padding1)self.conv2 nn.Conv2d(32, 64, kernel_size3, padding1)self.pool nn.MaxPool2d(kernel_size2, stride2, padding0)self.fc1 nn.Linear(64 * 16 * 16, 128)self.fc2 nn.Linear(128, 15)def forward(self, x):x self.pool(F.relu(self.conv1(x)))x self.pool(F.relu(self.conv2(x)))x x.view(-1, 64 * 16 * 16)x F.relu(self.fc1(x))x self.fc2(x)return x# 初始化模型、损失函数和优化器
model CNN()
criterion nn.CrossEntropyLoss()
optimizer optim.SGD(model.parameters(), lrlearning_rate, momentum0.9)# 训练模型
def train(model, train_loader, criterion, optimizer, epochs):model.train()running_loss 0.0for epoch in range(epochs):for data, target in tqdm(train_loader):optimizer.zero_grad()output model(data)loss criterion(output, target)loss.backward()optimizer.step()running_loss loss.item()print(fEpoch [{epoch 1}], Loss: {running_loss / len(train_loader):.4f})running_loss 0.0train(model, train_loader, criterion, optimizer, num_epochs)# 测试模型
def test(model, test_loader, criterion):model.eval()test_loss 0correct 0all_preds []all_targets []with torch.no_grad():for data, target in test_loader:output model(data)test_loss criterion(output, target).item()pred output.argmax(dim1, keepdimTrue)correct pred.eq(target.view_as(pred)).sum().item()all_preds.extend(pred.cpu().numpy())all_targets.extend(target.cpu().numpy())test_loss / len(test_loader.dataset)accuracy 100. * correct / len(test_loader.dataset)precision precision_score(all_targets, all_preds, averagemacro)recall recall_score(all_targets, all_preds, averagemacro)f1 f1_score(all_targets, all_preds, averagemacro)print(fTest Loss: {test_loss:.4f}, Accuracy: {accuracy:.2f}%, Precision: {precision:.4f}, Recall: {recall:.4f}, F1 Score: {f1:.4f})test(model, test_loader, criterion)
四、总结 本文基于汉字手写体数字图像进行了CNN分类实战CNN作为图像处理的经典模型展现出了它强大的图像特征提取能力结合更加复杂的模型框架CNN还可用于高精度人脸识别、物体识别等任务中。
