Generate captions for images

%pip install -qU pixeltable openai

import getpass
import os

if 'OPENAI_API_KEY' not in os.environ:
    os.environ['OPENAI_API_KEY'] = getpass.getpass('OpenAI API Key: ')

import pixeltable as pxt
from pixeltable.functions.openai import chat_completions

# Create a fresh directory
pxt.drop_dir('caption_demo', force=True)
pxt.create_dir('caption_demo')

# Create table for images
images = pxt.create_table('caption_demo/images', {'image': pxt.Image})

# Insert sample images
image_urls = [
    'https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/resources/images/000000000036.jpg',
    'https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/resources/images/000000000090.jpg',
    'https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/resources/images/000000000106.jpg',
]

images.insert([{'image': url} for url in image_urls])

# View images
images.collect()

# Add caption using OpenAI vision

messages = [
    {
        'role': 'user',
        'content': [
            {
                'type': 'text',
                'text': 'Write a concise, descriptive caption for this image in one sentence.',
            },
            {'type': 'image_url', 'image_url': images.image},
        ],
    }
]

images.add_computed_column(
    caption=chat_completions(messages, model='gpt-4o-mini')
)

# View images with captions
images.select(
    images.image, images.caption['choices'][0]['message']['content']
).collect()

# Add alt text for accessibility (brief)

messages = [
    {
        'role': 'user',
        'content': [
            {
                'type': 'text',
                'text': 'Write a brief alt text for this image (under 125 characters) for screen readers.',
            },
            {'type': 'image_url', 'image_url': images.image},
        ],
    }
]

images.add_computed_column(
    alt_text=chat_completions(messages, model='gpt-4o-mini')
)

# Add detailed description

messages = [
    {
        'role': 'user',
        'content': [
            {
                'type': 'text',
                'text': 'Describe this image in detail, including objects, colors, setting, and mood.',
            },
            {'type': 'image_url', 'image_url': images.image},
        ],
    }
]

images.add_computed_column(
    description=chat_completions(messages, model='gpt-4o-mini')
)

# View all caption types
images.select(
    images.image,
    images.caption['choices'][0]['message']['content'],
    images.alt_text['choices'][0]['message']['content'],
    images.description['choices'][0]['message']['content'],
).collect()