网站建设瀑布流,网上书城网站开发的目的与意义,龙岩app建设,wordpress打开有盗链CIFAR-10数据集在计算机视觉领域是一个很重要的数据集#xff0c;很有必要去熟悉它#xff0c;我们来到Kaggle站点#xff0c;进入到比赛页面#xff1a;https://www.kaggle.com/competitions/cifar-10CIFAR-10是8000万小图像数据集的一个子集#xff0c;由60000张32x32彩…CIFAR-10数据集在计算机视觉领域是一个很重要的数据集很有必要去熟悉它我们来到Kaggle站点进入到比赛页面https://www.kaggle.com/competitions/cifar-10CIFAR-10是8000万小图像数据集的一个子集由60000张32x32彩色图像组成包含10个分类每个类有6000张图像。官方数据中有5万张训练图片和1万张测试图片。我们保留了原始数据集中的训练/测试分割在Kaggle比赛提交的时候为了阻止某些形式的作弊(比如手标)我们在测试集中添加了29万张垃圾图片。这些图像在评分时被忽略。我们还对官方的10000个测试图像进行了微小的修改以防止通过文件散列查找它们。这些修改不应明显影响得分。您应该预测所有30万张图像的标签。对于刷排行榜这些我们不用去管秉持着学习为主的想法我们来训练这个数据集。分成10个类别分别为airplane、automobile、bird、cat、deer、dog、frog、horse、ship、truck这些类是完全相互排斥的比如汽车和卡车之间没有重叠。automobile包括轿车、suv之类的东西。truck只包括大卡车。这两项都不包括皮卡车。将下载的数据集放入到dataset目录解压之后在这个目录下面有train目录、test目录、trainLabels.csv标签文件其中train里面是5万张图片、test里面是30万张图片1、整理原始数据集1.1读取训练集的标签文件def read_label_file(data_dir,label_file,train_dir,valid_ratio):读取训练集的标签文件参数valid_ratio:验证集样本数与原始训练集样本数之比返回值n_train // len(labels):每个类多少张图片idx_label:50000个id:label的字典with open(os.path.join(data_dir,label_file),r) as f:linesf.readlines()[1:]tokens[l.rstrip().split(,) for l in lines]idx_labeldict(((int(idx),label) for idx,label in tokens))#{cat, ship, frog, dog, truck, deer, horse, bird, airplane, automobile}labelsset(idx_label.values())#去重就是10个类别n_train_validlen(os.listdir(os.path.join(data_dir,train_dir)))#50000n_trainint(n_train_valid*(1-valid_ratio))assert 0n_trainn_train_validreturn n_train // len(labels),idx_label我们测试下先熟悉下这个方法data_dir,label_filedataset,trainLabels.csv
train_dir,valid_ratiotrain,0.1n_train_per_label,idx_labelread_label_file(data_dir,label_file,train_dir,valid_ratio)
print(n_train_per_label,idx_label)#4500,{id:label,...}读取标签文件返回每个类有多少个训练样本(id:label这样的id对应标签的字典)1.2切分验证数据集上面读取标签的方法中有参数valid_ratio用来从原始训练集中切分出验证集这里设定为0.1接下来我们将切分的45000张图片用于训练5000张图片用于验证将它们分别存放到input_dir/train,input_dir/valid目录下面这里的input_dir我这里设置为train_valid_test在train目录下面是10个分类的目录(这个将定义一个方法自动创建)每个分类目录里面是4500张所属类别的图片在valid目录下面也是10个分类的目录(同样自动创建)每个分类目录里面是500张所属类别的图片还有一个train_valid目录下面同样是10个分类目录每个类别目录包含5000张图片。本人的路径如下D:\CIFAR10\dataset\train_valid_test\train\[airplane...]\[1-4500].pngD:\CIFAR10\dataset\train_valid_test\valid\[automobile...]\[1-500].pngD:\CIFAR10\dataset\train_valid_test\train_valid\[bird...]\[1-5000].png这里定义一个辅助函数新建不存在的路径将递归新建目录#辅助函数路径不存在就创建
def mkdir_if_not_exist(path):if not os.path.exists(os.path.join(*path)):os.makedirs(os.path.join(*path))def reorg_train_valid(data_dir,train_dir,input_dir,n_train_per_label,idx_label):切分训练数据集,分别生成train、valid、train_valid文件夹在这些目录下面分别生成10个类别目录,遍历图片拷贝到对应的类别目录label_count{}#{frog: 4500, cat: 4500, automobile: 4500,...}for train_file in os.listdir(os.path.join(data_dir,train_dir)):idxint(train_file.split(.)[0])labelidx_label[idx]#类别mkdir_if_not_exist([data_dir,input_dir,train_valid,label])src1os.path.join(data_dir,train_dir,train_file)dst1os.path.join(data_dir,input_dir,train_valid,label)shutil.copy(src1,dst1)#将图片拷贝到train_valid_test\train_valid\类别\if label not in label_count or label_count[label]n_train_per_label:mkdir_if_not_exist([data_dir,input_dir,train,label])src2os.path.join(data_dir,train_dir,train_file)dst2os.path.join(data_dir,input_dir,train,label)shutil.copy(src2,dst2)label_count[label]label_count.get(label,0)1#每个类别数量累加小于n_train_per_label4500else:mkdir_if_not_exist([data_dir,input_dir,valid,label])src3os.path.join(data_dir,train_dir,train_file)dst3os.path.join(data_dir,input_dir,valid,label)shutil.copy(src3,dst3)input_dirtrain_valid_test
reorg_train_valid(data_dir,train_dir,input_dir,n_train_per_label,idx_label)这个图片数量比较多拷贝过程比较耗时所以我们可以使用进度条来显示我们拷贝的进展。from tqdm import tqdmwith tqdm(totallen(os.listdir(os.path.join(data_dir,train_dir)))) as pbar:for train_file in tqdm(os.listdir(os.path.join(data_dir,train_dir))):......更多关于进度条的知识可以参阅Python中tqdm进度条的详细介绍(安装程序与耗时的迭代最终结果是训练数据集的图片都拷贝到了各自所对应类别的目录里面。1.3整理测试数据集训练与验证的数据集做好接下来做一个测试集用来预测的时候使用。def reorg_test(data_dir,test_dir,input_dir):mkdir_if_not_exist([data_dir,input_dir,test,unknown])for test_file in os.listdir(os.path.join(data_dir,test_dir)):srcos.path.join(data_dir,test_dir,test_file)dstos.path.join(data_dir,input_dir,test,unknown)shutil.copy(src,dst)reorg_test(data_dir,test,input_dir)这样就将dataset\test中的测试图片拷贝到了dataset\train_valid_test\test\unknown目录下面当然简单起见直接手动拷贝过去也可以。2、读取整理后的数据集2.1、图像增广为了应对过拟合我们使用图像增广关于图像增广在前面章节有讲过有兴趣的也可以查阅计算机视觉之图像增广(翻转、随机裁剪、颜色变化[亮度、对比度、饱和度、色调])这里我们将训练数据集做一些随机翻转、缩放裁剪与通道的标准化等处理对测试与验证数据集只做个标准化处理# 训练集图像增广
transform_train gdata.vision.transforms.Compose([gdata.vision.transforms.Resize(40),gdata.vision.transforms.RandomResizedCrop(32, scale(0.64, 1.0), ratio(1.0, 1.0)),gdata.vision.transforms.RandomFlipLeftRight(),gdata.vision.transforms.ToTensor(),gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465],[0.2023, 0.1994, 0.2010])])#测试集图像增广
transform_test gdata.vision.transforms.Compose([gdata.vision.transforms.ToTensor(),gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465],[0.2023, 0.1994, 0.2010])])2.2、读取数据集读取增广后的数据集使用ImageFolderDataset实例来读取整理之后的文件夹里的图片数据集其中每个数据样本包括图像和标签。#ImageFolderDataset加载存储在文件夹结构中的图像文件的数据集
train_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, train), flag1)
valid_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, valid), flag1)
train_valid_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, train_valid), flag1)
test_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, test), flag1)
print(train_ds.items[0:2],train_ds.items[-2:])[(dataset\\train_valid_test\\train\\airplane\\10009.png, 0), (dataset\\train_valid_test\\train\\airplane\\10011.png, 0)]
[(dataset\\train_valid_test\\train\\truck\\5235.png, 9), (dataset\\train_valid_test\\train\\truck\\5236.png, 9)]
打印的items来看返回的是列表里面的元素是元组对分别是图片路径与标签(类别)值。然后我们使用DataLoader实例指定增广之后的数据集返回小批量数据。在训练时我们仅用验证集评价模型因此需要保证输出的确定性。在预测时我们将在训练集和验证集的并集上训练模型以充分利用所有标注的数据。#DataLoader从数据集中加载数据并返回小批量数据
batch_size 128
train_iter gdata.DataLoader(train_ds.transform_first(transform_train), batch_size, shuffleTrue, last_batchkeep)
valid_iter gdata.DataLoader(valid_ds.transform_first(transform_test), batch_size, shuffleTrue, last_batchkeep)
train_valid_iter gdata.DataLoader(train_valid_ds.transform_first(transform_train), batch_size, shuffleTrue, last_batchkeep)
test_iter gdata.DataLoader(test_ds.transform_first(transform_test), batch_size, shuffleFalse, last_batchkeep)3、定义模型数据集处理好了之后我们就可以开始定义合适的模型了我们选用残差网络ResNet-18模型在此之前我们先使用基于HybridBlock类构建残差块#定义残差块
class Residual(nn.HybridBlock):def __init__(self, num_channels, use_1x1convFalse, strides1, **kwargs):super(Residual, self).__init__(**kwargs)self.conv1 nn.Conv2D(num_channels, kernel_size3,padding1, stridesstrides)self.conv2 nn.Conv2D(num_channels, kernel_size3, padding1)if use_1x1conv:self.conv3 nn.Conv2D(num_channels, kernel_size1, stridesstrides)else:self.conv3 Noneself.bn1 nn.BatchNorm()self.bn2 nn.BatchNorm()def hybrid_forward(self, F, X):Y F.relu(self.bn1(self.conv1(X)))Y self.bn2(self.conv2(Y))if self.conv3:X self.conv3(X)return F.relu(YX)
定义好了残差块就可以方便的构建残差网络了。#ResNet-18模型
def resnet18(num_classes):net nn.HybridSequential()net.add(nn.Conv2D(64, kernel_size3, strides1, padding1),nn.BatchNorm(), nn.Activation(relu))def resnet_block(num_channels, num_residuals, first_blockFalse):blk nn.HybridSequential()for i in range(num_residuals):if i 0 and not first_block:blk.add(Residual(num_channels, use_1x1convTrue, strides2))else:blk.add(Residual(num_channels))return blknet.add(resnet_block(64, 2, first_blockTrue), resnet_block(128, 2), resnet_block(256, 2), resnet_block(512, 2))net.add(nn.GlobalAvgPool2D(), nn.Dense(num_classes))return net定义好了模型在训练之前我们使用Xavier随机初始化我们这里是CIFAR10数据集有10个分类所以最终的稠密层我们输出的是10def get_net(ctx):num_classes 10net resnet18(num_classes)net.initialize(ctxctx, initinit.Xavier())return net
lossgloss.SoftmaxCrossEntropyLoss()4、训练模型模型初始化好了之后就可以对其进行训练了定义一个训练函数traindef train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay):trainer gluon.Trainer(net.collect_params(), sgd, {learning_rate: lr, momentum: 0.9, wd: wd})for epoch in range(num_epochs):train_l_sum, train_acc_sum, n, start 0.0, 0.0, 0, time.time()if epoch 0 and epoch % lr_period 0:trainer.set_learning_rate(trainer.learning_rate*lr_decay)for X, y in train_iter:y y.astype(float32).as_in_context(ctx)with autograd.record():y_hat net(X.as_in_context(ctx))l loss(y_hat, y).sum()l.backward()trainer.step(batch_size)train_l_sum l.asscalar()train_acc_sum (y_hat.argmax(axis1) y).sum().asscalar()n y.sizetime_s time %.2f sec % (time.time()-start)if valid_iter is not None:# 评估给定数据集上模型的准确性《使用验证集》valid_acc d2l.evaluate_accuracy(valid_iter, net, ctx)epoch_s (epoch %d,loss %f,train acc %f,valid acc %f, %(epoch1, train_l_sum/n, train_acc_sum/n, valid_acc))else:epoch_s (epoch %d,loss %f,train acc %f, %(epoch1, train_l_sum/n, train_acc_sum/n))print(epoch_stime_s,lr str(trainer.learning_rate))定义好了train函数就可以进行训练了# 开始训练
ctx, num_epochs, lr, wd d2l.try_gpu(), 1, 0.1, 5e-4
lr_period, lr_decay, net 80, 0.1, get_net(ctx)
net.hybridize()
train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay)这里我们可以简单的将num_epochs设置为1只迭代一次看下程序有没有什么bug与运行的怎么样epoch 1,loss 2.033364,train acc 0.294133,valid acc 0.345600,time 288.89 sec,lr 0.1运行是没有什么问题接下来就正式进入到分类的主题了5、测试集分类模型训练没有什么问题超参数什么的也设置好了我们使用所有训练数据集(包括验证集)重新训练模型对测试集进行分类这里我使用5个迭代来看下效果会是怎么样的num_epochs, preds 5, []
net.hybridize()
train(net, train_valid_iter, None, num_epochs,lr, wd, ctx, lr_period, lr_decay)
for X, _ in test_iter:y_hat net(X.as_in_context(ctx))preds.extend(y_hat.argmax(axis1).astype(int).asnumpy())
sorted_ids list(range(1, len(test_ds)1))
sorted_ids.sort(keylambda x: str(x))
df pd.DataFrame({id: sorted_ids, label: preds})
df[label] df[label].apply(lambda x: train_valid_ds.synsets[x])
df.to_csv(submission.csv, indexFalse)
epoch 1,loss 2.192931,train acc 0.253960,time 346.49 sec,lr 0.1
epoch 2,loss 1.663164,train acc 0.390080,time 118.79 sec,lr 0.1
epoch 3,loss 1.493299,train acc 0.456140,time 118.91 sec,lr 0.1
epoch 4,loss 1.356744,train acc 0.509440,time 117.40 sec,lr 0.1
epoch 5,loss 1.235666,train acc 0.556580,time 114.41 sec,lr 0.1
可以看到损失在降低精度在增加一切正常训练完毕将生成一个提交文件submission.csv然后将这个submission.csv文件提交看下打分与排名当然这里可以将迭代次数调大准确度也是会上来的我迭代了100次然后提交看下分数如何结果还是不错的附上全部代码import pandas as pd
import d2lzh as d2l
import os
from mxnet import autograd,gluon,init
from mxnet.gluon import data as gdata,loss as gloss,nn
import shutil
import timedef read_label_file(data_dir,label_file,train_dir,valid_ratio):读取训练集的标签文件参数valid_ratio:验证集样本数与原始训练集样本数之比返回值n_train // len(labels):每个类多少张图片idx_label:50000个id:label的字典with open(os.path.join(data_dir,label_file),r) as f:linesf.readlines()[1:]tokens[l.rstrip().split(,) for l in lines]idx_labeldict(((int(idx),label) for idx,label in tokens))#{cat, ship, frog, dog, truck, deer, horse, bird, airplane, automobile}labelsset(idx_label.values())#去重就是10个类别n_train_validlen(os.listdir(os.path.join(data_dir,train_dir)))#50000n_trainint(n_train_valid*(1-valid_ratio))assert 0n_trainn_train_validreturn n_train // len(labels),idx_labeldata_dir,label_filedataset,trainLabels.csv
train_dir,valid_ratiotrain,0.1n_train_per_label,idx_labelread_label_file(data_dir,label_file,train_dir,valid_ratio)
#print(n_train_per_label,len(idx_label))#辅助函数路径不存在就创建
def mkdir_if_not_exist(path):if not os.path.exists(os.path.join(*path)):os.makedirs(os.path.join(*path))def reorg_train_valid(data_dir,train_dir,input_dir,n_train_per_label,idx_label):切分训练数据集,分别生成train、valid、train_valid文件夹在这些目录下面分别生成10个类别目录,遍历图片拷贝到对应的类别目录label_count{}#{frog: 4500, cat: 4500, automobile: 4500,...}from tqdm import tqdmwith tqdm(totallen(os.listdir(os.path.join(data_dir,train_dir)))) as pbar:for train_file in tqdm(os.listdir(os.path.join(data_dir,train_dir))):idxint(train_file.split(.)[0])labelidx_label[idx]#类别mkdir_if_not_exist([data_dir,input_dir,train_valid,label])src1os.path.join(data_dir,train_dir,train_file)dst1os.path.join(data_dir,input_dir,train_valid,label)#shutil.copy(src1,dst1)#将图片拷贝到train_valid_test\train_valid\类别\if label not in label_count or label_count[label]n_train_per_label:mkdir_if_not_exist([data_dir,input_dir,train,label])src2os.path.join(data_dir,train_dir,train_file)dst2os.path.join(data_dir,input_dir,train,label)#shutil.copy(src2,dst2)label_count[label]label_count.get(label,0)1#每个类别数量累加小于n_train_per_label4500else:mkdir_if_not_exist([data_dir,input_dir,valid,label])src3os.path.join(data_dir,train_dir,train_file)dst3os.path.join(data_dir,input_dir,valid,label)#shutil.copy(src3,dst3)input_dirtrain_valid_test
#reorg_train_valid(data_dir,train_dir,input_dir,n_train_per_label,idx_label)def reorg_test(data_dir,test_dir,input_dir):mkdir_if_not_exist([data_dir,input_dir,test,unknown])for test_file in os.listdir(os.path.join(data_dir,test_dir)):srcos.path.join(data_dir,test_dir,test_file)dstos.path.join(data_dir,input_dir,test,unknown)#shutil.copy(src,dst)#reorg_test(data_dir,test,input_dir)# 训练集图像增广
transform_train gdata.vision.transforms.Compose([gdata.vision.transforms.Resize(40),gdata.vision.transforms.RandomResizedCrop(32, scale(0.64, 1.0), ratio(1.0, 1.0)),gdata.vision.transforms.RandomFlipLeftRight(),gdata.vision.transforms.ToTensor(),gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465],[0.2023, 0.1994, 0.2010])])#测试集图像增广
transform_test gdata.vision.transforms.Compose([gdata.vision.transforms.ToTensor(),gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465],[0.2023, 0.1994, 0.2010])])#读取增广后的数据集
#ImageFolderDataset加载存储在文件夹结构中的图像文件的数据集
train_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, train), flag1)
valid_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, valid), flag1)
train_valid_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, train_valid), flag1)
test_ds gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, test), flag1)
#print(train_ds.items[0:2],train_ds.items[-2:])[(dataset\\train_valid_test\\train\\airplane\\10009.png, 0), (dataset\\train_valid_test\\train\\airplane\\10011.png, 0)]
[(dataset\\train_valid_test\\train\\truck\\5235.png, 9), (dataset\\train_valid_test\\train\\truck\\5236.png, 9)]
#DataLoader从数据集中加载数据并返回小批量数据
batch_size 128
train_iter gdata.DataLoader(train_ds.transform_first(transform_train), batch_size, shuffleTrue, last_batchkeep)
valid_iter gdata.DataLoader(valid_ds.transform_first(transform_test), batch_size, shuffleTrue, last_batchkeep)
train_valid_iter gdata.DataLoader(train_valid_ds.transform_first(transform_train), batch_size, shuffleTrue, last_batchkeep)
test_iter gdata.DataLoader(test_ds.transform_first(transform_test), batch_size, shuffleFalse, last_batchkeep)#-----------------定义模型--------------------
#定义残差块
class Residual(nn.HybridBlock):def __init__(self, num_channels, use_1x1convFalse, strides1, **kwargs):super(Residual, self).__init__(**kwargs)self.conv1 nn.Conv2D(num_channels, kernel_size3, padding1, stridesstrides)self.conv2 nn.Conv2D(num_channels, kernel_size3, padding1)if use_1x1conv:self.conv3 nn.Conv2D(num_channels, kernel_size1, stridesstrides)else:self.conv3 Noneself.bn1 nn.BatchNorm()self.bn2 nn.BatchNorm()def hybrid_forward(self, F, X):Y F.relu(self.bn1(self.conv1(X)))Y self.bn2(self.conv2(Y))if self.conv3:X self.conv3(X)return F.relu(YX)#ResNet-18模型
def resnet18(num_classes):net nn.HybridSequential()net.add(nn.Conv2D(64, kernel_size3, strides1, padding1),nn.BatchNorm(), nn.Activation(relu))def resnet_block(num_channels, num_residuals, first_blockFalse):blk nn.HybridSequential()for i in range(num_residuals):if i 0 and not first_block:blk.add(Residual(num_channels, use_1x1convTrue, strides2))else:blk.add(Residual(num_channels))return blknet.add(resnet_block(64, 2, first_blockTrue), resnet_block(128, 2), resnet_block(256, 2), resnet_block(512, 2))net.add(nn.GlobalAvgPool2D(), nn.Dense(num_classes))return netdef get_net(ctx):num_classes 10net resnet18(num_classes)net.initialize(ctxctx, initinit.Xavier())return net
lossgloss.SoftmaxCrossEntropyLoss()#---------------------训练函数---------------------
def train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay):trainer gluon.Trainer(net.collect_params(), sgd, {learning_rate: lr, momentum: 0.9, wd: wd})for epoch in range(num_epochs):train_l_sum, train_acc_sum, n, start 0.0, 0.0, 0, time.time()if epoch 0 and epoch % lr_period 0:trainer.set_learning_rate(trainer.learning_rate*lr_decay)for X, y in train_iter:y y.astype(float32).as_in_context(ctx)with autograd.record():y_hat net(X.as_in_context(ctx))l loss(y_hat, y).sum()l.backward()trainer.step(batch_size)train_l_sum l.asscalar()train_acc_sum (y_hat.argmax(axis1) y).sum().asscalar()n y.sizetime_s time %.2f sec % (time.time()-start)if valid_iter is not None:# 评估给定数据集上模型的准确性《使用验证集》valid_acc d2l.evaluate_accuracy(valid_iter, net, ctx)epoch_s (epoch %d,loss %f,train acc %f,valid acc %f, %(epoch1, train_l_sum/n, train_acc_sum/n, valid_acc))else:epoch_s (epoch %d,loss %f,train acc %f, %(epoch1, train_l_sum/n, train_acc_sum/n))print(epoch_stime_s,lr str(trainer.learning_rate))# 开始训练
ctx, num_epochs, lr, wd d2l.try_gpu(), 1, 0.1, 5e-4
lr_period, lr_decay, net 80, 0.1, get_net(ctx)
#net.hybridize()
#train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay)num_epochs, preds 100, []
net.hybridize()
train(net, train_valid_iter, None, num_epochs,lr, wd, ctx, lr_period, lr_decay)
for X, _ in test_iter:y_hat net(X.as_in_context(ctx))preds.extend(y_hat.argmax(axis1).astype(int).asnumpy())
sorted_ids list(range(1, len(test_ds)1))
sorted_ids.sort(keylambda x: str(x))
df pd.DataFrame({id: sorted_ids, label: preds})
#apply应用synsets方法将0~9的数字分别转换为airplane、automobile...对应的类别
#synsets方法大家可以看定义就是获取文件夹名称(类别)
df[label] df[label].apply(lambda x: train_valid_ds.synsets[x])
df.to_csv(submission.csv, indexFalse)
