Using Label Studio for Annotations with Pixeltable

%pip install -qU pixeltable label-studio label-studio-sdk torch transformers

import subprocess
ls_process = subprocess.Popen(['label-studio'], stderr=subprocess.PIPE)

Performing system checks...

System check identified no issues (1 silenced).
August 14, 2024 - 04:24:46
Django version 3.2.25, using settings 'label_studio.core.settings.label_studio'
Starting development server at http://0.0.0.0:8080/
Quit the server with CONTROL-C.

import getpass
import os

if 'LABEL_STUDIO_URL' not in os.environ:
    os.environ['LABEL_STUDIO_URL'] = 'http://localhost:8080/'

if 'LABEL_STUDIO_API_KEY' not in os.environ:
    os.environ['LABEL_STUDIO_API_KEY'] = getpass.getpass('Label Studio API key: ')

Label Studio API key:  ········

import pixeltable as pxt

schema = {
    'video': pxt.Video,
    'date': pxt.Timestamp
}

# Before creating the table, we drop the `ls_demo` dir and all its contents,
# in order to ensure a clean environment for the demo.
pxt.drop_dir('ls_demo', force=True)
pxt.create_dir('ls_demo')
videos_table = pxt.create_table('ls_demo.videos', schema)

Connected to Pixeltable database at: postgresql://postgres:@/pixeltable?host=/Users/asiegel/.pixeltable/pgdata
Created directory `ls_demo`.
Created table `videos`.

from datetime import datetime

url_prefix = 'http://multimedia-commons.s3-website-us-west-2.amazonaws.com/data/videos/mp4/'
files = [
    '122/8ff/1228ff94bf742242ee7c88e4769ad5d5.mp4',
    '2cf/a20/2cfa205eae979b31b1144abd9fa4e521.mp4',
    'ffe/ff3/ffeff3c6bf57504e7a6cecaff6aefbc9.mp4',
]
today = datetime(2024, 4, 22)
videos_table.insert({'video': url_prefix + file, 'date': today} for file in files)

Inserting rows into `videos`: 3 rows [00:00, 993.05 rows/s]
Inserted 3 rows with 0 errors.

UpdateStatus(num_rows=3, num_computed_values=0, num_excs=0, updated_cols=[], cols_with_excs=[])

videos_table.head()

# Create a view to filter on the specified date

v = pxt.create_view(
    'ls_demo.videos_2024_04_22',
    videos_table.where(videos_table.date == today)
)

# Create a new Label Studio project and link it to the view. The
# configuration uses Label Studio's standard XML format. This only
# needs to be done once: after the view and project are linked,
# the relationship is stored indefinitely in Pixeltable's metadata.

label_config = '''
    <View>
      <Video name="video" value="$video"/>
      <Choices name="video-category" toName="video" showInLine="true">
        <Choice value="city"/>
        <Choice value="food"/>
        <Choice value="sports"/>
      </Choices>
    </View>
    '''

pxt.io.create_label_studio_project(v, label_config)

Inserting rows into `videos_2024_04_22`: 3 rows [00:00, 1864.69 rows/s]
Created view `videos_2024_04_22` with 3 rows, 0 exceptions.
Added 3 column values with 0 errors.
Computing cells: 100%|███████████████████████████████████████████| 3/3 [00:00<00:00, 857.44 cells/s]
Linked external store `ls_project_0` to table `videos_2024_04_22`.
Created 3 new task(s) in LabelStudioProject `videos_2024_04_22`.

SyncStatus(external_rows_created=3, external_rows_deleted=0, external_rows_updated=0, pxt_rows_updated=0, num_excs=0)

v

v = pxt.get_table('ls_demo.videos_2024_04_22')
v.sync()

Created 0 new task(s) in LabelStudioProject `videos_2024_04_22`.
Updated annotation(s) from 1 task(s) in LabelStudioProject `videos_2024_04_22`.

SyncStatus(external_rows_created=0, external_rows_deleted=0, external_rows_updated=0, pxt_rows_updated=1, num_excs=0)

v.select(v.video, v.annotations).head()

v.add_computed_column(
    video_category=v.annotations[0].result[0].value.choices[0]
)
v.select(v.video, v.annotations, v.video_category).head()

Computing cells: 100%|███████████████████████████████████████████| 3/3 [00:00<00:00, 394.63 cells/s]
Added 3 column values with 0 errors.

from pixeltable.functions.video import get_metadata

v.add_computed_column(video_metadata=get_metadata(v.video))
v.select(v.video, v.annotations, v.video_category, v.video_metadata).head()

Computing cells: 100%|███████████████████████████████████████████| 3/3 [00:00<00:00, 138.97 cells/s]
Added 3 column values with 0 errors.

from datetime import datetime
from pixeltable.iterators import FrameIterator

today = datetime(2024, 4, 22)
videos_table = pxt.get_table('ls_demo.videos')

# Create the view, using a `FrameIterator` to extract frames with a sample rate
# of `fps=0.25`, or 1 frame per 4 seconds of video. Setting `fps=0` would use the
# native framerate of the video, extracting every frame.

frames = pxt.create_view(
    'ls_demo.frames_2024_04_22',
    videos_table.where(videos_table.date == today),
    iterator=FrameIterator.create(video=videos_table.video, fps=0.25)
)

Inserting rows into `frames_2024_04_22`: 13 rows [00:00, 5434.66 rows/s]
Created view `frames_2024_04_22` with 13 rows, 0 exceptions.

# Show just the first 3 frames in the table, to avoid cluttering the notebook

frames.select(frames.frame).head(3)

from pixeltable.functions.huggingface import detr_for_object_detection

# Run the Resnet-50 object detection model against each frame to generate bounding boxes
frames.add_computed_column(detections=detr_for_object_detection(
    frames.frame,
    model_id='facebook/detr-resnet-50',
    threshold=0.95
)
frames.select(frames.frame, frames.detections).head(3)

Computing cells: 100%|██████████████████████████████████████████| 13/13 [00:06<00:00,  2.10 cells/s]
Added 13 column values with 0 errors.

from pixeltable.functions.huggingface import detr_to_coco

frames.add_computed_column(
    preannotations=detr_to_coco(frames.frame, frames.detections)
)
frames.select(frames.frame, frames.detections, frames.preannotations).head(3)

Computing cells: 100%|██████████████████████████████████████████| 13/13 [00:00<00:00, 42.79 cells/s]
Added 13 column values with 0 errors.

frames_config = '''
    <View>
      <Image name="frame" value="$frame"/>
      <RectangleLabels name="preannotations" toName="frame">
        <Label value="car" background="blue"/>
        <Label value="person" background="red"/>
        <Label value="train" background="green"/>
      </RectangleLabels>
    </View>
    '''

pxt.io.create_label_studio_project(frames, frames_config)

Added 13 column values with 0 errors.
Computing cells: 100%|██████████████████████████████████████████| 13/13 [00:00<00:00, 42.09 cells/s]
Linked external store `ls_project_0` to table `frames_2024_04_22`.
Created 13 new task(s) in LabelStudioProject `frames_2024_04_22`.

SyncStatus(external_rows_created=13, external_rows_deleted=0, external_rows_updated=0, pxt_rows_updated=0, num_excs=0)

videos_table.insert(
    video=url_prefix + '22a/948/22a9487a92956ac453a9c15e0fc4dd4.mp4',
    date=today
)

Inserting rows into `videos`: 1 rows [00:00, 808.31 rows/s]
Inserting rows into `videos_2024_04_22`: 1 rows [00:00, 849.57 rows/s]
Inserting rows into `frames_2024_04_22`: 5 rows [00:00, 3225.89 rows/s]
Inserted 7 rows with 0 errors.

UpdateStatus(num_rows=7, num_computed_values=0, num_excs=0, updated_cols=[], cols_with_excs=[])

v.sync()
frames.sync()

Created 1 new task(s) in LabelStudioProject `videos_2024_04_22`.
Created 5 new task(s) in LabelStudioProject `frames_2024_04_22`.

SyncStatus(external_rows_created=5, external_rows_deleted=0, external_rows_updated=0, pxt_rows_updated=0, num_excs=0)

v.external_stores  # Get a list of all external stores for `v`

['ls_project_0']

v.unlink_external_stores('ls_project_0', delete_external_data=True)

Deleted Label Studio project: videos_2024_04_22
Unlinked external store from table `videos_2024_04_22`: ls_project_0

pxt.io.create_label_studio_project(
    v,
    label_config,
    media_import_method='url',
    s3_configuration={'bucket': 'pxt-test', 'aws_access_key_id': my_key, 'aws_secret_access_key': my_secret}
)

ls_process.kill()