View source Download .ipynb

Import instance segmentation dataset from multiple image masks

In this tutorial, we will import the LIACi (Lifecycle Inspection, Analysis and Condition information) Segmentation Dataset for Underwater Ship Inspections, introduced in this paper.

The dataset contains 1893 images of underwater ship hulls, together with corresponding annotations. The dataset contains both COCO-style annotations (bounding boxes and segmentation polygons) and pixel-wise annotations stored as single-channel bitmap images with one image per class.

In this notebook, we will create two different tlc.Tables from the dataset, in order to showcase different ways of working with annotated image data in 3LC:

1. Build a 3LC Instance Segmentation Table using the individual per-class bitmaps.

2. Build a 3LC Instance Segmentation Table using the COCO-style annotations.

Since the downloaded data includes pre-computed embeddings, we will also add the embeddings to a Run, reduce the dimensionality of the embeddings and visualize the results.

Project setup

PROJECT_NAME = "3LC Tutorials"
DATASET_NAME = "LIACI"
INSTANCE_SEGMENTATION_TABLE_NAME = "instance-segmentation"
COCO_TABLE_NAME = "coco-segmentation"

Imports

import json
from pathlib import Path

import cv2
import numpy as np
import tlc
import tqdm

Prepare dataset

The dataset is available for download from the official website, and must be downloaded and extracted to a local directory manually.

The dataset is stored in the following layout:

LIACi_dataset_pretty
│
├── images
│   ├── image_0001.jpg
│   ├── image_0002.jpg
│   ├── image_0003.jpg
│   └── ...
│
├── masks
│   ├── anode
│   │   ├── image_0001.bmp
│   │   ├── image_0002.bmp
│   │   ├── image_0003.bmp
│   │   └── ...
│   ├── bilge_keel
│   ├── corrosion
│   ├── defect
│   ├── marine_growth
│   ├── over_board_valves
│   ├── paint_peel
│   ├── propeller
│   ├── saliency
│   ├── sea_chest_grating
│   ├── segmentation         # merged masks
│   └── ship_hull
│
├── coco-annotations.json
├── train_test_split.csv
├── embeddings_resnet101.json
...

# Replace with your own path, after downloading and extracting the dataset
DATASET_ROOT = Path("C:/Data/LIACi_dataset_pretty")

# Register the dataset root as an alias, enabling easy sharing/moving of the table
tlc.register_url_alias("LIACI_DATASET_ROOT", DATASET_ROOT.as_posix())

Approach 1: instance segmentations from per-class masks

image_dir = DATASET_ROOT / "images"
masks_dir = DATASET_ROOT / "masks"

# Exclude merged masks, we are only interested in per-class masks
mask_dirs = [d.name for d in masks_dir.iterdir() if d.is_dir() and d.name != "segmentation"]

We will now collect the data which will go into our Table. For the segmentation column, we will stack all the per-class masks into a single array.

image_urls = []
instance_segmentations = []

images = list(image_dir.iterdir())

for image in tqdm.tqdm(images, total=len(images), desc="Processing images"):
    image_url = tlc.Url(image).to_relative()  # .to_relative() applies the alias to the path
    mask_filename = image_url.name.replace("jpg", "bmp")

    image_urls.append(image_url.to_str())

    masks = []
    cat_ids = []

    for cat_id, category in enumerate(mask_dirs):
        mask_path = masks_dir / category / mask_filename
        if not mask_path.exists():
            print(f"Skipping category {category} for image {image_url}")
            continue

        mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)

        if not np.any(mask):
            # If the mask is empty, we skip the category
            continue

        mask[mask == 255] = 1
        masks.append(mask)
        cat_ids.append(cat_id)

    # Masks in 3LC instance segmentation masks format
    instance_segmentation = {
        "image_height": masks[0].shape[0],
        "image_width": masks[0].shape[1],
        "instance_properties": {
            "label": cat_ids,
        },
        "masks": np.stack(masks, axis=-1),
    }
    instance_segmentations.append(instance_segmentation)

Create a TableWriter to write the data to the Table with the correct column schemas.

table_writer = tlc.TableWriter(
    table_name=INSTANCE_SEGMENTATION_TABLE_NAME,
    project_name=PROJECT_NAME,
    dataset_name=DATASET_NAME,
    column_schemas={
        "image": tlc.ImagePath("image"),
        "segmentations": tlc.InstanceSegmentationMasks(
            "segmentations",
            instance_properties_structure={
                "label": tlc.CategoricalLabel("label", mask_dirs),
            },
        ),
    },
)

table_writer.add_batch(
    {
        "image": image_urls,
        "segmentations": instance_segmentations,
    }
)

instance_segmentation_table = table_writer.finalize()

sample_masks = instance_segmentation_table[0]["segmentations"]["masks"]
mask_labels = instance_segmentation_table[0]["segmentations"]["instance_properties"]["label"]

import matplotlib.pyplot as plt

fig, axes = plt.subplots(2, 3, figsize=(12, 8))
for i in range(2):
    for j in range(3):
        idx = i * 3 + j
        axes[i, j].imshow(sample_masks[:, :, idx], cmap="gray")
        axes[i, j].set_title(f"{mask_dirs[mask_labels[idx]]}")
        axes[i, j].axis("off")
plt.tight_layout()
plt.show()

Approach 2: COCO-style annotations

annotations_path = DATASET_ROOT / "coco-labels.json"
image_folder = DATASET_ROOT / "images"

coco_table = tlc.Table.from_coco(
    annotations_path,
    image_folder,
    project_name=PROJECT_NAME,
    dataset_name=DATASET_NAME,
    table_name=COCO_TABLE_NAME,
    task="segment",
    segmentation_format="masks",
)

coco_table.rows_schema["segmentations"].sample_type

Prepare data for YOLO (return polygons and split into train/test)

from tlc_tools.derived_tables import masks_to_polygons
from tlc_tools.split import split_table

polygons_table = masks_to_polygons(coco_table)
splits = split_table(polygons_table)

print(f"Train split: {splits['train']}")
print(f"Validation split: {splits['val']}")

Extra: Reduce and visualize embeddings

# Load the pre-computed embeddings (2048-dimensional)
embeddings_json_path = Path(DATASET_ROOT) / "embeddings_resnet101.json"

with open(embeddings_json_path) as f:
    embeddings_json = json.load(f)

Create a run to store the embeddings.

run = tlc.init(
    PROJECT_NAME,
    "LIACi-Embeddings",
    description="Inspect 2D and 3D embeddings of the LIACi dataset",
)

Link the rows of the embeddings metrics table to the instance segmentation table, using the foreign_table_url parameter.

foreign_table_url = tlc.Url.create_table_url(
    INSTANCE_SEGMENTATION_TABLE_NAME,
    DATASET_NAME,
    PROJECT_NAME,
)

embedding_table_writer = tlc.MetricsTableWriter(
    run_url=run.url,
    foreign_table_url=foreign_table_url,
    column_schemas={
        "embedding": tlc.Schema(
            value=tlc.Float32Value(number_role=tlc.NUMBER_ROLE_NN_EMBEDDING),
            size0=tlc.DimensionNumericValue(value_min=2048, value_max=2048),
            writable=False,
        )
    },
)

Add the original embeddings to the Run.

embeddings = embeddings_json["embeddings"]
for i, embedding in tqdm.tqdm(enumerate(embeddings.values()), total=len(embeddings), delay=1.0):
    embedding_table_writer.add_row({tlc.EXAMPLE_ID: i, "embedding": embedding})

embedding_table = embedding_table_writer.finalize()

Reduce the dimensionality of the embeddings to 3D. Keep the original embeddings in the table.

reduced_3d = tlc.reduce_embeddings(
    embedding_table,
    method="pacmap",
    n_components=3,
    n_neighbors=20,
    MN_ratio=0.7,
    FP_ratio=3.0,
    target_embedding_column="embedding_3d",
    retain_source_embedding_column=True,  # We need the source embedding column for the 2D reduction
)

reduced_3d.columns

Reduce the dimensionality of the embeddings to 2D. Delete the original embeddings from the table.

reduced_2d = tlc.reduce_embeddings(
    reduced_3d,
    method="pacmap",
    n_components=2,
    n_neighbors=20,
    MN_ratio=0.7,
    FP_ratio=3.0,
    target_embedding_column="embedding_2d",
    retain_source_embedding_column=False,  # We don't need the source embedding column anymore
)

Add the metrics Table to the Run.

run.add_metrics_table(reduced_2d)

run.set_status_completed()