HuggingFaceM4/pmd

Both are exclusive. "image_url": datasets.Value("string"), "image": datasets.Image(), # An image can have multiple text associated with the same value. For example COCO. "texts": [datasets.Value("string")], # Define where the sample comes from, this is necessary when we start to use aggregated versions like PMD. "source": datasets.Value("string"), # We commit any kind of additional information in json format in `meta` "meta": datasets.Value("string"), } ) def json_serializer(o): if isinstance(o, datetime): return str(o) raise TypeError(f"Object of type {o.__class__.__name__} is not JSON serializable") class BaseLoader: def __init__(self, source: str, split: str, writer_batch_size: int): self.source = source self.split = split self.writer_batch_size = writer_batch_size @abstractmethod def _generate_batches(self): raise NotImplementedError() class DatasetsLoader(BaseLoader):