Error when processing speech_commands dataset

[guillaumelorre28]

/!\ PLEASE INCLUDE THE FULL STACKTRACE AND CODE SNIPPET

Short description
An error occurs when processing the speech_commands dataset.

Environment information

• Operating System: mac0S
• Python version: Python 3.10
• tensorflow-datasets/tfds-nightly version: 4.9.4
• tensorflow/tf-nightly version: 2.16.1

Reproduction instructions

import tensorflow_datasets as tfds

dataset_name = "speech_commands"
tfds_path = "/Users/glo/tensorflow-datasets"  # Change the path

builder = tfds.builder(dataset_name, data_dir=tfds_path)
builder.download_and_prepare()

Link to logs

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/tensorflow_datasets/core/dataset_builder.py:1584, in GeneratorBasedBuilder._download_and_prepare(self, dl_manager, download_config)
1572 for split_name, generator in utils.tqdm(
1573 split_generators.items(),
1574 desc="Generating splits...",
1575 unit=" splits",
1576 leave=False,
1577 ):
1578 filename_template = naming.ShardedFileTemplate(
1579 split=split_name,
1580 dataset_name=self.name,
1581 data_dir=self.data_path,
1582 filetype_suffix=path_suffix,
1583 )
-> 1584 future = split_builder.submit_split_generation(
1585 split_name=split_name,
1586 generator=generator,
1587 filename_template=filename_template,
1588 disable_shuffling=self.info.disable_shuffling,
1589 )
1590 split_info_futures.append(future)
1592 # Process the result of the beam pipeline.

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/tensorflow_datasets/core/split_builder.py:341, in SplitBuilder.submit_split_generation(self, split_name, generator, filename_template, disable_shuffling)
338 # Depending on the type of generator, we use the corresponding
339 # _build_from_xyz method.
340 if isinstance(generator, collections.abc.Iterable):
--> 341 return self._build_from_generator(**build_kwargs)
342 else: # Otherwise, beam required
343 unknown_generator_type = TypeError(
344 f'Invalid split generator value for split {split_name}. '
345 'Expected generator or apache_beam object. Got: '
346 f'{type(generator)}'
347 )

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/tensorflow_datasets/core/split_builder.py:406, in SplitBuilder._build_from_generator(self, split_name, generator, filename_template, disable_shuffling)
396 serialized_info = self._features.get_serialized_info()
397 writer = writer_lib.Writer(
398 serializer=example_serializer.ExampleSerializer(serialized_info),
399 filename_template=filename_template,
(...)
404 shard_config=self._shard_config,
405 )
--> 406 for key, example in utils.tqdm(
407 generator,
408 desc=f'Generating {split_name} examples...',
409 unit=' examples',
410 total=total_num_examples,
411 leave=False,
412 mininterval=1.0,
413 ):
414 try:
415 example = self._features.encode_example(example)

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/tqdm/notebook.py:249, in tqdm_notebook.iter(self)
247 try:
248 it = super(tqdm_notebook, self).iter()
--> 249 for obj in it:
250 # return super(tqdm...) will not catch exception
251 yield obj
252 # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/tqdm/std.py:1182, in tqdm.iter(self)
1179 time = self._time
1181 try:
-> 1182 for obj in iterable:
1183 yield obj
1184 # Update and possibly print the progressbar.
1185 # Note: does not call self.update(1) for speed optimisation.

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/tensorflow_datasets/datasets/speech_commands/speech_commands_dataset_builder.py:138, in Builder._generate_examples(self, archive, file_list)
134 else:
135 try:
136 example = {
137 'audio': np.array(
--> 138 lazy_imports_lib.lazy_imports.pydub.AudioSegment.from_file(
139 file_obj, format='wav'
140 ).get_array_of_samples()
141 ),
142 'label': label,
143 }
144 yield example_id, example
145 except (
146 lazy_imports_lib.lazy_imports.pydub.exceptions.CouldntDecodeError
147 ):

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/pydub/audio_segment.py:728, in AudioSegment.from_file(cls, file, format, codec, parameters, start_second, duration, **kwargs)
726 info = None
727 else:
--> 728 info = mediainfo_json(orig_file, read_ahead_limit=read_ahead_limit)
729 if info:
730 audio_streams = [x for x in info['streams']
731 if x['codec_type'] == 'audio']

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/site-packages/pydub/utils.py:279, in mediainfo_json(filepath, read_ahead_limit)
276 output = output.decode("utf-8", 'ignore')
277 stderr = stderr.decode("utf-8", 'ignore')
--> 279 info = json.loads(output)
281 if not info:
282 # If ffprobe didn't give any information, just return it
283 # (for example, because the file doesn't exist)
284 return info

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/json/init.py:346, in loads(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
341 s = s.decode(detect_encoding(s), 'surrogatepass')
343 if (cls is None and object_hook is None and
344 parse_int is None and parse_float is None and
345 parse_constant is None and object_pairs_hook is None and not kw):
--> 346 return _default_decoder.decode(s)
347 if cls is None:
348 cls = JSONDecoder

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/json/decoder.py:337, in JSONDecoder.decode(self, s, _w)
332 def decode(self, s, _w=WHITESPACE.match):
333 """Return the Python representation of s (a str instance
334 containing a JSON document).
335
336 """
--> 337 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
338 end = _w(s, end).end()
339 if end != len(s):

File ~/opt/anaconda3/envs/tensorflow_latest/lib/python3.10/json/decoder.py:355, in JSONDecoder.raw_decode(self, s, idx)
353 obj, end = self.scan_once(s, idx)
354 except StopIteration as err:
--> 355 raise JSONDecodeError("Expecting value", s, err.value) from None
356 return obj, end

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

Expected behavior
No error during the processing of the speech_command dataset.

[fylux]

Did you also get this error without a custom data dir? I cannot reproduce the error running in Colab the following:

import tensorflow_datasets as tfds

dataset_name = "speech_commands"

builder = tfds.builder(dataset_name)
builder.download_and_prepare()

[Toby1218]

I reproduced the same problem. I was able to fix it by downgrading pydub to version 0.23.1 instead of 0.25.1