package main

import (
	"fmt"
	"io"
	"net/http"
	"time"

	warc "github.com/internetarchive/gowarc"
	"github.com/miekg/dns"
)

func main() {
	// Configure WARC settings
	rotatorSettings := &warc.RotatorSettings{
		WarcinfoContent: warc.Header{
			"software": "My WARC writing client v1.0",
		},
		Prefix:             "WEB",
		Compression:        "gzip",
		WARCWriterPoolSize: 1, // Records will be written to 4 WARC files in parallel, it helps maximize the disk IO on some hardware. To be noted, even if we have multiple WARC writers, WARCs are ALWAYS written by pair in the same file. (req/resp pair)
	}

	// Configure DNS fallback settings
	dnsFallback := &dns.ClientConfig{
		Servers:  []string{"1.1.1.1", "1.0.0.1"}, // Cloudflare DNS
		Port:     "53",
		Ndots:    1,
		Timeout:  5,
		Attempts: 1,
	}

	// Configure HTTP client settings
	clientSettings := warc.HTTPClientSettings{
		RotatorSettings: rotatorSettings,
		TempDir:         "./temp",
		DNSServers:      []string{"8.8.8.8", "8.8.4.4"},
		DNSFallback:     dnsFallback, // Fallback DNS config if /etc/resolv.conf is unavailable
		DedupeOptions: warc.DedupeOptions{
			LocalDedupe:   true,
			CDXDedupe:     false,
			SizeThreshold: 2048, // Only payloads above that threshold will be deduped
		},
		DialTimeout:           10 * time.Second,
		ResponseHeaderTimeout: 30 * time.Second,
		DNSResolutionTimeout:  5 * time.Second,
		DNSRecordsTTL:         5 * time.Minute,
		DNSCacheSize:          10000,
		MaxReadBeforeTruncate: 1000000000,
		DecompressBody:        true,
		FollowRedirects:       true,
		VerifyCerts:           true,
		RandomLocalIP:         true,
	}

	// Create HTTP client
	client, err := warc.NewWARCWritingHTTPClient(clientSettings)
	if err != nil {
		panic(err)
	}
	defer client.Close()

	// The error channel NEED to be consumed, else it will block the
	// execution of the WARC module
	go func() {
		for err := range client.ErrChan {
			fmt.Errorf("WARC writer error: %s", err.Err.Error())
		}
	}()

	// This is optional but the module give a feedback on a channel passed as context value to the
	// request, this helps knowing when the record has been written to disk. If this is not used, the WARC
	// writing is asynchronous
	req, err := http.NewRequest("GET", "https://archive.org/robots.txt", nil)
	if err != nil {
		panic(err)
	}

	feedbackChan := make(chan struct{}, 1)
	req = req.WithContext(warc.WithFeedbackChannel(req.Context(), feedbackChan))

	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	// Process response
	// Note: the body NEED to be consumed to be written to the WARC file.
	io.Copy(io.Discard, resp.Body)

	// Will block until records are actually written to the WARC file
	<-feedbackChan
}