Module `hub.core.chunk_engine.tests.test_full_benchmark_chunk_engine`

Expand source code

from typing import Tuple

import numpy as np
import pytest

from hub.constants import GB
from hub.core.chunk_engine import read_array, write_array
from hub.core.chunk_engine.tests.common import (
    get_random_array,
    TENSOR_KEY,
)
from hub.core.tests.common import parametrize_all_caches, parametrize_all_storages
from hub.core.typing import StorageProvider
from hub.tests.common_benchmark import (
    parametrize_full_benchmark_shapes,
    parametrize_benchmark_chunk_sizes,
    parametrize_benchmark_dtypes,
    parametrize_benchmark_num_batches,
)

from .test_benchmark_chunk_engine import (
    single_benchmark_write,
    benchmark_write,
    single_benchmark_read,
    benchmark_read,
)


@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_write_with_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_caches
def test_write_with_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_write(
        benchmark=benchmark,
        shape=shape,
        dtype=dtype,
        chunk_size=chunk_size,
        num_batches=num_batches,
        storage=storage,
    )


@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_write_without_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_storages
def test_write_without_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_write(
        benchmark=benchmark,
        shape=shape,
        dtype=dtype,
        chunk_size=chunk_size,
        num_batches=num_batches,
        storage=storage,
    )


@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_read_with_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_caches
def test_read_with_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_read(benchmark, shape, dtype, chunk_size, num_batches, storage)


@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_read_without_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_storages
def test_read_without_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_read(benchmark, shape, dtype, chunk_size, num_batches, storage)

Functions

def test_read_with_caches(benchmark, shape: Tuple[int], chunk_size: int, num_batches: int, dtype: str, storage: StorageProvider)

Expand source code

@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_read_with_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_caches
def test_read_with_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_read(benchmark, shape, dtype, chunk_size, num_batches, storage)

def test_read_without_caches(benchmark, shape: Tuple[int], chunk_size: int, num_batches: int, dtype: str, storage: StorageProvider)

Expand source code

@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_read_without_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_storages
def test_read_without_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_read(benchmark, shape, dtype, chunk_size, num_batches, storage)

def test_write_with_caches(benchmark, shape: Tuple[int], chunk_size: int, num_batches: int, dtype: str, storage: StorageProvider)

Expand source code

@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_write_with_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_caches
def test_write_with_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_write(
        benchmark=benchmark,
        shape=shape,
        dtype=dtype,
        chunk_size=chunk_size,
        num_batches=num_batches,
        storage=storage,
    )

def test_write_without_caches(benchmark, shape: Tuple[int], chunk_size: int, num_batches: int, dtype: str, storage: StorageProvider)

Expand source code

@pytest.mark.full_benchmark
@pytest.mark.benchmark(group="chunk_engine_write_without_caches_FULL")
@parametrize_full_benchmark_shapes
@parametrize_benchmark_num_batches
@parametrize_benchmark_chunk_sizes
@parametrize_benchmark_dtypes
@parametrize_all_storages
def test_write_without_caches(
    benchmark,
    shape: Tuple[int],
    chunk_size: int,
    num_batches: int,
    dtype: str,
    storage: StorageProvider,
):
    benchmark_write(
        benchmark=benchmark,
        shape=shape,
        dtype=dtype,
        chunk_size=chunk_size,
        num_batches=num_batches,
        storage=storage,
    )