{
  "topic": "huggingface",
  "title": "Hugging Face Transformers / Hub / Diffusers",
  "claim": "Operational notes for loading, tokenizing, generating, Hub cache, and Diffusers memory.",
  "notes": [
    {
      "slug": "hf-chat-templates",
      "title": "Chat templates in Hugging Face Transformers",
      "url": "https://iinventedlight.com/library/notes/hf-chat-templates/",
      "status": "published"
    },
    {
      "slug": "hf-dataset-viewer-api",
      "title": "Using the Dataset Viewer API",
      "url": "https://iinventedlight.com/library/notes/hf-dataset-viewer-api/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-arrow-memory-mapping",
      "title": "Why Hugging Face Datasets uses Apache Arrow and memory mapping",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-arrow-memory-mapping/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-audio-loading",
      "title": "Load audio datasets with Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-audio-loading/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-audio-preprocess",
      "title": "Process audio datasets: resample and extract features",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-audio-preprocess/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-audiofolder",
      "title": "Create an audio dataset with AudioFolder",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-audiofolder/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-batch-map",
      "title": "Batch mapping can change a dataset row count",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-batch-map/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-cache",
      "title": "How the Hugging Face Datasets cache works and where it lives",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-cache/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-cloud-storage",
      "title": "Move dataset files from cloud storage with fsspec",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-cloud-storage/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-concatenate-interleave",
      "title": "Concatenate and interleave Hugging Face datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-concatenate-interleave/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-create-dataset",
      "title": "Create a Hugging Face dataset from your own files",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-create-dataset/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-dataset-card",
      "title": "Write a dataset card on the Hugging Face Hub",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-dataset-card/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-depth-estimation",
      "title": "Prepare depth estimation datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-depth-estimation/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-faiss-search",
      "title": "Add a FAISS search index to a Hugging Face dataset",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-faiss-search/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-features-casting",
      "title": "Hugging Face Datasets features: column types, ClassLabel and cast",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-features-casting/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-image-loading",
      "title": "Load image datasets with Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-image-loading/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-image-preprocess",
      "title": "Image preprocessing separates cached work from access-time transforms",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-image-preprocess/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-imagefolder",
      "title": "ImageFolder derives labels from folders and joins metadata by file name",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-imagefolder/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-jax",
      "title": "JAX formatting happens when Datasets examples are read",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-jax/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-load-and-stream",
      "title": "Loading and streaming Hugging Face datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-load-and-stream/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-load-from-hub",
      "title": "Loading Hub data separates revision, files, and splits",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-load-from-hub/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-load-local-files",
      "title": "Load local CSV, JSON or Parquet files with Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-load-local-files/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-map-filter",
      "title": "Hugging Face Datasets: map, filter, select, and other row and column operations",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-map-filter/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-map-style-vs-iterable",
      "title": "Dataset and IterableDataset differ in access and evaluation",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-map-style-vs-iterable/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-numpy-format",
      "title": "NumPy formatting controls dataset reads, not the stored table",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-numpy-format/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-object-detection",
      "title": "Detection transforms must update boxes along with images",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-object-detection/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-pandas",
      "title": "Pandas formatting and conversion serve different uses",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-pandas/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-polars",
      "title": "Polars batches can process Datasets without row-by-row Python",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-polars/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-preprocess-text",
      "title": "Tokenize a text dataset with Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-preprocess-text/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-pyarrow",
      "title": "Hugging Face Datasets and Apache Arrow",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-pyarrow/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-pytorch",
      "title": "Use Hugging Face Datasets with PyTorch",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-pytorch/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-repository-structure",
      "title": "Hub repository structure declares splits and configurations",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-repository-structure/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-semantic-segmentation",
      "title": "Prepare semantic segmentation datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-semantic-segmentation/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-share-cli",
      "title": "The hf CLI uploads dataset files with an explicit repository type",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-share-cli/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-sort-shuffle-select",
      "title": "Sort, shuffle and select rows in Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-sort-shuffle-select/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-spark",
      "title": "Load a Spark DataFrame into Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-spark/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-split-slicing",
      "title": "Load part of a split in Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-split-slicing/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-tensorflow",
      "title": "Use Hugging Face Datasets with TensorFlow and Keras",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-tensorflow/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-troubleshoot",
      "title": "Datasets troubleshooting starts with the failed operation",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-troubleshoot/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-upload",
      "title": "Share a dataset on the Hugging Face Hub",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-upload/",
      "status": "published"
    },
    {
      "slug": "hf-datasets-video-loading",
      "title": "Load video datasets with Hugging Face Datasets",
      "url": "https://iinventedlight.com/library/notes/hf-datasets-video-loading/",
      "status": "published"
    },
    {
      "slug": "hf-diffusers-memory",
      "title": "Reducing diffusers memory: offloading, device_map, VAE slicing",
      "url": "https://iinventedlight.com/library/notes/hf-diffusers-memory/",
      "status": "published"
    },
    {
      "slug": "hf-diffusers-pipeline-loading",
      "title": "DiffusionPipeline: components, subclasses, dtype, device_map",
      "url": "https://iinventedlight.com/library/notes/hf-diffusers-pipeline-loading/",
      "status": "published"
    },
    {
      "slug": "hf-diffusers-schedulers",
      "title": "Diffusers schedulers: choosing, swapping, timestep schedules",
      "url": "https://iinventedlight.com/library/notes/hf-diffusers-schedulers/",
      "status": "published"
    },
    {
      "slug": "hf-gated-repos",
      "title": "Gated models and datasets on the Hugging Face Hub",
      "url": "https://iinventedlight.com/library/notes/hf-gated-repos/",
      "status": "published"
    },
    {
      "slug": "hf-hub-cache-management",
      "title": "The Hugging Face Hub cache: layout, location, cleanup",
      "url": "https://iinventedlight.com/library/notes/hf-hub-cache-management/",
      "status": "published"
    },
    {
      "slug": "hf-hub-download-files",
      "title": "Download files from the Hugging Face Hub: hf_hub_download and snapshot_download",
      "url": "https://iinventedlight.com/library/notes/hf-hub-download-files/",
      "status": "published"
    },
    {
      "slug": "hf-hub-inference-client",
      "title": "InferenceClient: one client for providers, endpoints, local servers",
      "url": "https://iinventedlight.com/library/notes/hf-hub-inference-client/",
      "status": "published"
    },
    {
      "slug": "hf-hub-repos-dataset-cards",
      "title": "Hub repositories and dataset cards",
      "url": "https://iinventedlight.com/library/notes/hf-hub-repos-dataset-cards/",
      "status": "published"
    },
    {
      "slug": "hf-hub-upload-files",
      "title": "Uploading to the Hub with huggingface_hub",
      "url": "https://iinventedlight.com/library/notes/hf-hub-upload-files/",
      "status": "published"
    },
    {
      "slug": "hf-model-cards",
      "title": "Hugging Face model cards: README.md plus YAML metadata",
      "url": "https://iinventedlight.com/library/notes/hf-model-cards/",
      "status": "published"
    },
    {
      "slug": "hf-safetensors",
      "title": "Safetensors stores tensor data without pickle object loading",
      "url": "https://iinventedlight.com/library/notes/hf-safetensors/",
      "status": "published"
    },
    {
      "slug": "hf-text-generation-strategies",
      "title": "Text generation strategies in Transformers",
      "url": "https://iinventedlight.com/library/notes/hf-text-generation-strategies/",
      "status": "published"
    },
    {
      "slug": "hf-tokenization-subword-algorithms",
      "title": "Subword tokenization: BPE, byte-level BPE, Unigram, WordPiece",
      "url": "https://iinventedlight.com/library/notes/hf-tokenization-subword-algorithms/",
      "status": "published"
    },
    {
      "slug": "hf-tokenizers-padding-truncation",
      "title": "Tokenizer padding, truncation, and special tokens",
      "url": "https://iinventedlight.com/library/notes/hf-tokenizers-padding-truncation/",
      "status": "published"
    },
    {
      "slug": "hf-trainer",
      "title": "Fine-tuning with the Transformers Trainer",
      "url": "https://iinventedlight.com/library/notes/hf-trainer/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-attention-backends",
      "title": "Attention backends: sdpa, FlashAttention, flex, paged",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-attention-backends/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-auto-classes",
      "title": "AutoModel classes and from_pretrained",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-auto-classes/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-big-model-loading",
      "title": "Loading big models: device_map, meta device, shards",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-big-model-loading/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-generate-pitfalls",
      "title": "generate() pitfalls: length, greedy default, padding side, prompt format",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-generate-pitfalls/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-kv-cache",
      "title": "KV cache in Transformers: why it exists and which Cache class",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-kv-cache/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-pipeline",
      "title": "Hugging Face Transformers pipeline(): inference in a few lines",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-pipeline/",
      "status": "published"
    },
    {
      "slug": "hf-transformers-quantization-overview",
      "title": "Quantization in Transformers: load-time vs pre-quantized",
      "url": "https://iinventedlight.com/library/notes/hf-transformers-quantization-overview/",
      "status": "published"
    }
  ],
  "artifact_refs": [
    "/library/artifacts/hf-load-choices.table.json",
    "/library/artifacts/hf-from-pretrained.schema.json"
  ],
  "confidence": "medium",
  "as_of": "2026-11-09",
  "license": "Index of CC-BY-4.0 library notes; underlying source licenses remain on each note",
  "schema": "https://iinventedlight.com/library/schemas/topic-packet.schema.json"
}