-
Notifications
You must be signed in to change notification settings - Fork 1
/
createNewDataset.py
133 lines (105 loc) · 3.83 KB
/
createNewDataset.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#!/usr/bin/env python
# Author:
# Rohit Sharma (srohit0.github.io)
#
# Date:
# June 15, 2019
#
# Description:
# This script creates smaller dataset out of food-101 dataset available at
# http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz
#
# Download and untar the dataset in current directory before using this script.
# % curl http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz
# % tar xvz food-101.tar.gz
import os, sys, errno, shutil
import json
nClasses = 10
dsSrc = 'food-101'
dsDest = 'food-' + str(nClasses)
def mkdir_p(path):
try:
os.mkdir(path)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
# Write
# 1. class file,
# 2. label text file
# 3. label json file
# 4. copy images
def writeDatasetFiles(dsSrc, dsDest, metaTag, classes):
print "Writing " + str(nClasses) + " " + metaTag + " classes.";
dsDestLableFile = dsDest + "/meta/" + metaTag + ".txt"
dsDestClassFile = dsDest + "/meta/classes.txt"
with open(dsDestLableFile, 'w') as dlf, open(dsDestClassFile, 'w') as dcf:
for cls in sorted(classes.keys()):
# write class
dcf.write(cls+"\n");
for lbl in classes[cls]:
# write label
dlf.write(cls+"/"+lbl+"\n");
# copy image
mkdir_p(dsDest+"/images/"+cls)
imageFile = "/images/" + cls + "/" + lbl + ".jpg"
shutil.copy2(dsSrc+imageFile, dsDest+imageFile)
# Write Json file
dsDestJsonFile = dsDest + "/meta/" + metaTag + ".json"
with open(dsDestJsonFile, 'w') as djf:
json.dump(classes, djf)
return;
def createNewDataset(dsSrc, dsDest, training_classes, test_classes):
mkdir_p(dsDest);
shutil.copy2(dsSrc+"/license_agreement.txt", dsDest+"/license_agreement.txt")
shutil.copy2(dsSrc+"/README.txt", dsDest+"/README.txt")
mkdir_p(dsDest + str("/images"));
mkdir_p(dsDest + str("/meta"));
writeDatasetFiles(dsSrc, dsDest, "train", training_classes)
writeDatasetFiles(dsSrc, dsDest, "test", test_classes)
def genLabels(labelFile, newLabelFile, nClasses, nSamplesPerClass):
classes = {}
with open(labelFile) as f:
content = [x.strip() for x in f.readlines()]
iClass = 0
iSamples = 1
for label in content:
cls, lbl = os.path.split(label)
if ( classes.has_key(cls) and iSamples >= nSamplesPerClass ):
continue;
if classes.has_key(cls):
classes[cls].append(lbl);
iSamples = iSamples + 1
else:
classes[cls] = [lbl]
iClass = iClass + 1
iSamples = 1
if ( iClass >= nClasses and iSamples >= nSamplesPerClass):
break
return classes
def parseClasses(dsSrc, dsDest):
training_classes = genLabels(
dsSrc + str("/meta/train.txt"),
dsDest + str("/meta/train.txt"),
nClasses, 375
)
test_classes = genLabels(
dsSrc + str("/meta/test.txt"),
dsDest + str("/meta/test.txt"),
nClasses, 125
)
return training_classes, test_classes
def checkPythonVersion(rv):
currentVersion = sys.version_info
if currentVersion[0] == rv[0] and currentVersion[1] >= rv[1]:
pass
else:
sys.stderr.write( "\n%s - Error: Python interpreter must be %d.%d or greater (within major version %d)\n" % (sys.argv[0], rv[0], rv[1], rv[0]) )
sys.exit(-1)
return 0
if __name__ == "__main__":
checkPythonVersion( (2,1) )
training_classes, test_classes = parseClasses(dsSrc, dsDest);
createNewDataset(dsSrc, dsDest, training_classes, test_classes)
sys.exit(0)