package main

import (
    "fmt"
    "log"

    "github.com/tsawler/tabula"
)

func main() {
    // Works with PDF, DOCX, ODT, XLSX, PPTX, HTML, and EPUB files
    text, warnings, err := tabula.Open("document.pdf").Text()
    // text, warnings, err := tabula.Open("document.docx").Text()
    // text, warnings, err := tabula.Open("document.odt").Text()
    // text, warnings, err := tabula.Open("spreadsheet.xlsx").Text()
    // text, warnings, err := tabula.Open("presentation.pptx").Text()
    // text, warnings, err := tabula.Open("page.html").Text()
    // text, warnings, err := tabula.Open("book.epub").Text()
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println(text)

    for _, w := range warnings {
        fmt.Println("Warning:", w.Message)
    }
}

// PDF with all options
text, warnings, err := tabula.Open("document.pdf").
    Pages(1, 2, 3).              // Specific pages (PDF only)
    ExcludeHeadersAndFooters().  // Remove headers/footers
    JoinParagraphs().            // Join text into paragraphs (PDF only)
    Text()

// DOCX/ODT with header/footer exclusion
text, warnings, err := tabula.Open("document.docx").
    ExcludeHeadersAndFooters().  // Remove headers/footers
    Text()

// XLSX (each sheet extracted as tab-separated values)
text, warnings, err := tabula.Open("spreadsheet.xlsx").Text()

// PPTX (each slide extracted with title and content)
text, warnings, err := tabula.Open("presentation.pptx").
    ExcludeHeadersAndFooters().  // Remove slide footers and numbers
    Text()

// HTML (extracts text from headings, paragraphs, lists, tables)
// Use navigation exclusion to remove headers, footers, and nav elements
text, warnings, err := tabula.Open("page.html").Text()

// HTML from URL with navigation filtering
import "github.com/tsawler/tabula/htmldoc"
resp, _ := http.Get("https://example.com")
reader, _ := htmldoc.OpenReader(resp.Body)
opts := htmldoc.ExtractOptions{NavigationExclusion: htmldoc.NavigationExclusionStandard}
text, _ := reader.TextWithOptions(opts)

// EPUB (e-books, supports EPUB 2 and EPUB 3)
text, warnings, err := tabula.Open("book.epub").Text()

// PDF with header/footer exclusion
markdown, warnings, err := tabula.Open("document.pdf").
    ExcludeHeadersAndFooters().
    ToMarkdown()

// DOCX with header/footer exclusion (preserves headings, lists, tables)
markdown, warnings, err := tabula.Open("document.docx").
    ExcludeHeadersAndFooters().
    ToMarkdown()

// ODT with header/footer exclusion (preserves headings, lists, tables)
markdown, warnings, err := tabula.Open("document.odt").
    ExcludeHeadersAndFooters().
    ToMarkdown()

// XLSX (each sheet as a markdown table)
markdown, warnings, err := tabula.Open("spreadsheet.xlsx").ToMarkdown()

// PPTX (each slide with title as heading, content, and tables)
markdown, warnings, err := tabula.Open("presentation.pptx").
    ExcludeHeadersAndFooters().
    ToMarkdown()

// HTML (preserves headings, lists, tables, code blocks)
// Use navigation exclusion to remove headers, footers, and nav elements
markdown, warnings, err := tabula.Open("page.html").ToMarkdown()

// HTML with aggressive navigation filtering
import "github.com/tsawler/tabula/htmldoc"
reader, _ := htmldoc.OpenReader(resp.Body)
opts := htmldoc.ExtractOptions{NavigationExclusion: htmldoc.NavigationExclusionAggressive}
markdown, _ := reader.MarkdownWithOptions(opts)

// EPUB (preserves chapter structure, headings, lists)
markdown, warnings, err := tabula.Open("book.epub").ToMarkdown()

package main

import (
    "fmt"
    "log"

    "github.com/tsawler/tabula"
)

func main() {
    // Works with PDF, DOCX, ODT, XLSX, PPTX, HTML, and EPUB
    chunks, warnings, err := tabula.Open("document.pdf").Chunks()
    // chunks, warnings, err := tabula.Open("document.docx").Chunks()
    // chunks, warnings, err := tabula.Open("document.odt").Chunks()
    // chunks, warnings, err := tabula.Open("spreadsheet.xlsx").Chunks()
    // chunks, warnings, err := tabula.Open("presentation.pptx").Chunks()
    // chunks, warnings, err := tabula.Open("page.html").Chunks()
    // chunks, warnings, err := tabula.Open("book.epub").Chunks()
    if err != nil {
        log.Fatal(err)
    }

    for i, chunk := range chunks.Chunks {
        fmt.Printf("Chunk %d: %s (p.%d-%d, ~%d tokens)\n",
            i+1,
            chunk.Metadata.SectionTitle,
            chunk.Metadata.PageStart,
            chunk.Metadata.PageEnd,
            chunk.Metadata.EstimatedTokens)
        fmt.Println(chunk.Text)
        fmt.Println("---")
    }

    // Warnings are non-fatal issues
    for _, w := range warnings {
        fmt.Println("Warning:", w.Message)
    }
}

chunks, _, err := tabula.Open("document.pdf").
    ExcludeHeadersAndFooters().
    Chunks()
if err != nil {
    log.Fatal(err)
}

// Get each chunk as separate markdown strings
mdChunks := chunks.ToMarkdownChunks()

for i, md := range mdChunks {
    // Store each chunk in your vector database
    embedding := embedModel.Embed(md)
    vectorDB.Store(chunks.Chunks[i].ID, embedding, md)
}

// From file path (format auto-detected by extension)
ext := tabula.Open("document.pdf")
ext := tabula.Open("document.docx")
ext := tabula.Open("document.odt")
ext := tabula.Open("spreadsheet.xlsx")
ext := tabula.Open("presentation.pptx")
ext := tabula.Open("page.html")
ext := tabula.Open("book.epub")

// From existing PDF reader (PDF only)
r, _ := reader.Open("document.pdf")
ext := tabula.FromReader(r)

// From HTML string (useful for web scraping)
html := `<html><body><h1>Hello</h1><p>World</p></body></html>`
ext := tabula.FromHTMLString(html)

// From HTML io.Reader (useful for HTTP responses)
resp, _ := http.Get("https://example.com/page")
defer resp.Body.Close()
ext := tabula.FromHTMLReader(resp.Body)

ext := tabula.Open("document.pdf")
defer ext.Close()

isCharLevel, _ := ext.IsCharacterLevel()  // Detect character-level PDFs
isMultiCol, _ := ext.IsMultiColumn()      // Detect multi-column layouts
pageCount, _ := ext.PageCount()           // Get page count (works with DOCX and ODT too)

import "github.com/tsawler/tabula/htmldoc"

// From HTTP response
resp, _ := http.Get("https://example.com/article")
defer resp.Body.Close()
reader, _ := htmldoc.OpenReader(resp.Body)

// Choose exclusion mode
opts := htmldoc.ExtractOptions{
    NavigationExclusion: htmldoc.NavigationExclusionStandard,
}

// Extract clean text or markdown
text, _ := reader.TextWithOptions(opts)
markdown, _ := reader.MarkdownWithOptions(opts)

// Works automatically on scanned PDFs (when built with -tags ocr)
text, warnings, err := tabula.Open("scanned-document.pdf").Text()
if err != nil {
    log.Fatal(err)
}

// Check if OCR was used
for _, w := range warnings {
    if w.Code == tabula.WarningOCRFallback {
        fmt.Println("Note: OCR was used for some pages")
    }
}

chunks, _, _ := tabula.Open("doc.pdf").Chunks()

// Filter by content type
tablesOnly := chunks.FilterWithTables()
listsOnly := chunks.FilterWithLists()

// Filter by location
section := chunks.FilterBySection("Introduction")
page5 := chunks.FilterByPage(5)
pages1to10 := chunks.FilterByPageRange(1, 10)

// Filter by size
smallChunks := chunks.FilterByMaxTokens(500)
largeChunks := chunks.FilterByMinTokens(100)

// Search
matches := chunks.Search("keyword")

// Chain filters
result := chunks.
    FilterBySection("Methods").
    FilterByMinTokens(100).
    Search("algorithm")

import "github.com/tsawler/tabula/rag"

// Options supported by all formats (PDF, DOCX, ODT, XLSX, PPTX, HTML, EPUB)
opts := rag.MarkdownOptions{
    IncludeMetadata:        true,   // YAML front matter with document metadata
    IncludeTableOfContents: true,   // Generated TOC from headings
    HeadingLevelOffset:     0,      // Adjust heading levels (1 makes H1 -> H2)
    MaxHeadingLevel:        6,      // Cap heading depth
}

// Works with all formats
markdown, _, _ := tabula.Open("doc.pdf").ToMarkdownWithOptions(opts)
markdown, _, _ := tabula.Open("doc.docx").ToMarkdownWithOptions(opts)
markdown, _, _ := tabula.Open("doc.odt").ToMarkdownWithOptions(opts)
markdown, _, _ := tabula.Open("spreadsheet.xlsx").ToMarkdownWithOptions(opts)
markdown, _, _ := tabula.Open("presentation.pptx").ToMarkdownWithOptions(opts)
markdown, _, _ := tabula.Open("page.html").ToMarkdownWithOptions(opts)
markdown, _, _ := tabula.Open("book.epub").ToMarkdownWithOptions(opts)

// PDF-only options (used via RAG chunking pipeline)
pdfOpts := rag.MarkdownOptions{
    IncludeMetadata:        true,
    IncludeChunkSeparators: true,   // --- between chunks (PDF only)
    IncludePageNumbers:     true,   // Page references (PDF only)
    IncludeChunkIDs:        true,   // HTML comments with chunk IDs (PDF only)
}

// Or use preset for RAG
opts := rag.RAGOptimizedMarkdownOptions()

import "github.com/tsawler/tabula/rag"

config := rag.ChunkerConfig{
    TargetChunkSize: 500,   // Target characters per chunk
    MaxChunkSize:    1000,  // Maximum characters
    MinChunkSize:    100,   // Minimum characters
    OverlapSize:     50,    // Overlap between chunks
}
sizeConfig := rag.DefaultSizeConfig()

chunks, _, _ := tabula.Open("doc.pdf").ChunksWithConfig(config, sizeConfig)

for _, chunk := range chunks.Chunks {
    fmt.Println("ID:", chunk.ID)
    fmt.Println("Section:", chunk.Metadata.SectionTitle)
    fmt.Println("Pages:", chunk.Metadata.PageStart, "-", chunk.Metadata.PageEnd)
    fmt.Println("Words:", chunk.Metadata.WordCount)
    fmt.Println("Tokens:", chunk.Metadata.EstimatedTokens)
    fmt.Println("Has Table:", chunk.Metadata.HasTable)
    fmt.Println("Has List:", chunk.Metadata.HasList)
}

stats := chunks.Statistics()
fmt.Println("Total chunks:", stats.TotalChunks)
fmt.Println("Total words:", stats.TotalWords)
fmt.Println("Average tokens:", stats.AvgTokens)
fmt.Println("Chunks with tables:", stats.ChunksWithTables)

text, warnings, err := tabula.Open("document.pdf").Text()
if err != nil {
    log.Fatal(err)  // Fatal error
}

for _, w := range warnings {
    log.Println("Warning:", w.Message)  // Non-fatal issues
}

// Format all warnings
formatted := tabula.FormatWarnings(warnings)

// Panic on error (for scripts/tests)
text := tabula.MustText(tabula.Open("doc.pdf").Text())
count := tabula.Must(tabula.Open("doc.pdf").PageCount())