Module `hub.api.tests.test_api`

Expand source code

import os, shutil
import numpy as np
from hub.api.dataset import Dataset
from hub.core.storage import LocalProvider
from hub.core.chunk_engine.read import read_dataset_meta, read_tensor_meta
from hub.core.tests.common import parametrize_all_dataset_storages

import pytest


def test_persist_local(local_storage):
    if local_storage is None:
        pytest.skip()

    ds = Dataset(local_storage.root)
    ds["image"] = np.ones((4, 4096, 4096))

    ds_new = Dataset(local_storage.root)
    assert len(ds_new) == 4
    assert ds_new["image"].shape == (4096, 4096)
    np.testing.assert_array_equal(ds_new["image"].numpy(), np.ones((4, 4096, 4096)))


@parametrize_all_dataset_storages
def test_populate_dataset(ds):
    assert read_dataset_meta(ds.provider) == {"tensors": []}
    ds["image"] = np.ones((4, 28, 28))
    assert read_dataset_meta(ds.provider) == {"tensors": ["image"]}
    assert len(ds) == 4


@parametrize_all_dataset_storages
def test_compute_tensor(ds):
    ds["image"] = np.ones((32, 28, 28))
    np.testing.assert_array_equal(ds["image"].numpy(), np.ones((32, 28, 28)))


@parametrize_all_dataset_storages
def test_compute_tensor_slice(ds):
    ds["image"] = np.vstack((np.arange(16),) * 8)

    sliced_data = ds["image"][2:5].numpy()
    expected_data = np.vstack((np.arange(16),) * 3)
    np.testing.assert_array_equal(sliced_data, expected_data)


@parametrize_all_dataset_storages
def test_iterate_dataset(ds):
    labels = [1, 9, 7, 4]
    ds["image"] = np.ones((4, 28, 28))
    ds["label"] = np.asarray(labels).reshape((4, 1))

    for idx, sub_ds in enumerate(ds):
        img = sub_ds["image"].numpy()
        label = sub_ds["label"].numpy()
        np.testing.assert_array_equal(img, np.ones((1, 28, 28)))
        assert label.shape == (1, 1)
        assert label[0] == labels[idx]

Functions

def test_compute_tensor(ds)

Expand source code

@parametrize_all_dataset_storages
def test_compute_tensor(ds):
    ds["image"] = np.ones((32, 28, 28))
    np.testing.assert_array_equal(ds["image"].numpy(), np.ones((32, 28, 28)))

def test_compute_tensor_slice(ds)

Expand source code

@parametrize_all_dataset_storages
def test_compute_tensor_slice(ds):
    ds["image"] = np.vstack((np.arange(16),) * 8)

    sliced_data = ds["image"][2:5].numpy()
    expected_data = np.vstack((np.arange(16),) * 3)
    np.testing.assert_array_equal(sliced_data, expected_data)

def test_iterate_dataset(ds)

Expand source code

@parametrize_all_dataset_storages
def test_iterate_dataset(ds):
    labels = [1, 9, 7, 4]
    ds["image"] = np.ones((4, 28, 28))
    ds["label"] = np.asarray(labels).reshape((4, 1))

    for idx, sub_ds in enumerate(ds):
        img = sub_ds["image"].numpy()
        label = sub_ds["label"].numpy()
        np.testing.assert_array_equal(img, np.ones((1, 28, 28)))
        assert label.shape == (1, 1)
        assert label[0] == labels[idx]

def test_persist_local(local_storage)

Expand source code

def test_persist_local(local_storage):
    if local_storage is None:
        pytest.skip()

    ds = Dataset(local_storage.root)
    ds["image"] = np.ones((4, 4096, 4096))

    ds_new = Dataset(local_storage.root)
    assert len(ds_new) == 4
    assert ds_new["image"].shape == (4096, 4096)
    np.testing.assert_array_equal(ds_new["image"].numpy(), np.ones((4, 4096, 4096)))

def test_populate_dataset(ds)

Expand source code

@parametrize_all_dataset_storages
def test_populate_dataset(ds):
    assert read_dataset_meta(ds.provider) == {"tensors": []}
    ds["image"] = np.ones((4, 28, 28))
    assert read_dataset_meta(ds.provider) == {"tensors": ["image"]}
    assert len(ds) == 4