真正进行工程数据训练，肯定没有事先处理好的数据集供机器训练，以下是实验室写的一段图片预处理的函数。是将一个图片数据集处理为tensorflow可以处理运算的数据类型。数据集整理了五个文件夹，每个文件夹内含有240左右的精灵图片，将它们转化为Dataset格式。
该实验案例的研究数据下载地址：链接: https://pan.baidu.com/s/1jszHjIPYSw7nTrY_ogSiLA 提取码: sura
如果你有其他非图片数据要处理遇到困难，欢迎沟通交流，因为是自学，希望结交更多志趣相同的同学。喜欢科学计算 统计学的可以加微信 wangqime
import os, csv, random
import tensorflow as tf
from tensorflow import keras
# 构建文件夹编码
dict = {}
def get_file_code(value):
if value in dict.keys():
return dict[value]
dict[value] = len(dict.keys())
return dict[value]
# 图片数据预处理为tensor类型
def preprocess(x, y):
x = tf.io.read_file(x)
x = tf.image.decode_jpeg(x, channels=3)
x = tf.image.resize(x, [224, 224])
x = tf.cast(x, dtype=tf.float32) / 255
# y = list(map(int, y))
y = tf.convert_to_tensor(y)
return x, y
# 构建获取图片路径的列表
def get_imgpath():
imgdata = []
imglabel = []
for root, dirs, files in os.walk('./pokemon', topdown=True, onerror=None, followlinks=False):
for dir in dirs:
dir_path = os.path.join(root, dir)
for roots, dirname, filelist in os.walk(dir_path):
for file in filelist:
# 获取图片
filepath = os.path.join(roots, file)
# 获取图片编码
fileencode = get_file_code(roots)
# 得到图片与对应编码的列表
imgdata.append(filepath)
imglabel.append(fileencode)
return imgdata, imglabel
# 写入csv保存制作训练数据集
def get_points_x_y():
rows = []
(x, y) = get_imgpath()
for i in range(len(x)):
row = [x[i], y[i]]
rows.append(row)
random.shuffle(rows)
with open('data.csv', mode='w', encoding='utf-8') as f:
f_csv = csv.writer(f)
res = rows
f_csv.writerows(res)
with open('data.csv', mode='r', encoding='utf-8') as f:
f_csv = csv.reader(f)
imgs = []
labels = []
for rows in f_csv:
img = rows[0]
label = int(rows[1])
imgs.append(img)
labels.append(label)
return imgs, labels
def get_data_type(imgs, labels, mode='train'):
if mode == 'train':
x = imgs[: int(0.6 * len(imgs))]
y = labels[: int(0.6 * len(labels))]
elif mode == 'val':
x = imgs[int(0.6 * len(imgs)):int(0.8 * len(imgs))]
y = labels[int(0.6 * len(imgs)):int(0.8 * len(imgs))]
else:
x = imgs[int(0.8 * len(imgs)):]
y = labels[int(0.8 * len(labels)):]
return x, y
def main():
get_imgpath()
img, label = get_points_x_y()
# print(img, label)
x, y = get_data_type(img, label, mode='train')
db = tf.data.Dataset.from_tensor_slices((x, y))
db = db.shuffle(1000).map(preprocess).batch(32)
print(db)
# 构建图片训练模型
#后续更新模型函数
#model = tf.keras.Sequential()
if __name__ == '__main__':
main()该段函数是输出训练集，如果需要验证集和测试集合更改对应的mode参数即可，打印结果如下：
(<tf.Tensor: id=36, shape=(32, 224, 224, 3), dtype=float32, numpy= array([[[[0.45882353, 0.7647059 , 0.6627451 ], [0.45882353, 0.7647059 , 0.6627451 ], [0.45882353, 0.7647059 , 0.6627451 ], ..., [0.3882353 , 0.7019608 , 0.5882353 ], [0.3882353 , 0.7019608 , 0.5882353 ], [0.38431373, 0.69803923, 0.58431375]], [[0.45882353, 0.76862746, 0.654902 ], [0.45882353, 0.76862746, 0.654902 ], [0.45882353, 0.76862746, 0.654902 ], ..., [0.39215687, 0.7058824 , 0.5921569 ], [0.39215687, 0.7058824 , 0.5921569 ], [0.3882353 , 0.7019608 , 0.5882353 ]], [[0.46092436, 0.7707283 , 0.6570028 ], [0.4630252 , 0.7728291 , 0.65910363], [0.4630252 , 0.7728291 , 0.65910363], ..., [0.39635855, 0.70980394, 0.5864146 ], [0.39635855, 0.70980394, 0.5864146 ], [0.39635855, 0.70980394, 0.5864146 ]], ..., [[0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], ..., [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805]], [[0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], ..., [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805]], [[0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], ..., [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805], [0.68235296, 0.8156863 , 0.34509805]]], [[[0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], ..., [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ]], [[0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], ..., [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ]], [[0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], ..., [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ]], ..., [[0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], ..., [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ]], [[0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], ..., [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ]], [[0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], ..., [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ], [0.96862745, 0.9882353 , 0.9647059 ]]], [[[0.24593839, 0.23655465, 0.26154965], [0.24929973, 0.24103644, 0.264861 ], [0.24313729, 0.23935577, 0.25849843], ..., [0.25882354, 0.21568628, 0.2784314 ], [0.25882354, 0.21568628, 0.2784314 ], [0.2579834 , 0.21484615, 0.27759123]], [[0.2910214 , 0.2404912 , 0.31659666], [0.29374248, 0.2435124 , 0.31946778], [0.2902711 , 0.24124148, 0.31659666], ..., [0.25882354, 0.21568628, 0.2784314 ], [0.25882354, 0.21568628, 0.2784314 ], [0.2579834 , 0.21484615, 0.27759123]], [[0.28847036, 0.24005602, 0.32289916], [0.28305322, 0.23606943, 0.31835234], [0.28411865, 0.24285714, 0.32289916], ..., [0.25882354, 0.21568628, 0.2784314 ], [0.2509804 , 0.20784314, 0.27058825], [0.2557421 , 0.21260484, 0.27534994]], ..., [[0.11155503, 0.09194719, 0.1154766 ], [0.12941177, 0.10980392, 0.13333334], [0.14130154, 0.12169372, 0.13737997], ..., [0.08802562, 0.03312366, 0.08410405], [0.08704427, 0.03781499, 0.07703067], [0.05406183, 0.02661085, 0.0579834 ]], [[0.09530778, 0.07569994, 0.10315092], [0.10623215, 0.08662431, 0.11407529], [0.11365043, 0.09404258, 0.11862276], ..., [0.07766066, 0.03844497, 0.07373909], [0.0730397 , 0.02598087, 0.06934886], [0.04789896, 0.02044798, 0.04789896]], [[0.06855715, 0.0489493 , 0.07640029], [0.08177726, 0.06216942, 0.0896204 ], [0.08956555, 0.0699577 , 0.09740868], ..., [0.08557368, 0.046358 , 0.08165211], [0.07820612, 0.0311473 , 0.07820612], [0.05014026, 0.02268928, 0.05014026]]], ..., [[[0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], ..., [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ]], [[0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], ..., [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ]], [[0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], ..., [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ], [0.08627451, 0.5529412 , 0.6862745 ]], ..., [[0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], ..., [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], [0.68617445, 0.64695877, 0.917547 ]], [[0.6837534 , 0.6445377 , 0.91512597], [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], ..., [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], [0.6846535 , 0.64543784, 0.9160261 ]], [[0.6843335 , 0.6451178 , 0.9157061 ], [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], ..., [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706], [0.6862745 , 0.64705884, 0.91764706]]], [[[1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ], ..., [1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ]], [[1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ], ..., [1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ]], [[1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ], ..., [1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ]], ..., [[1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ], ..., [1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ]], [[1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ], ..., [1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ]], [[1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ], ..., [1. , 1. , 1. ], [1. , 1. , 1. ], [1. , 1. , 1. ]]], [[[0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ], ..., [0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ]], [[0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ], ..., [0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ]], [[0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ], ..., [0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ]], ..., [[0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ], ..., [0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ]], [[0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ], ..., [0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ]], [[0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ], ..., [0. , 0. , 0. ], [0. , 0. , 0. ], [0. , 0. , 0. ]]]], dtype=float32)>, <tf.Tensor: id=37, shape=(32,), dtype=int32, numpy= array([4, 0, 3, 1, 4, 0, 0, 4, 0, 2, 2, 4, 0, 4, 1, 2, 2, 0, 1, 3, 2, 2, 4, 4, 2, 3, 2, 3, 0, 1, 0, 4], dtype=int32)>)
网络世界,不加微信QQ手机,留言沟通
发表评论