# Step 1: Download Common Crawl WARC files from Amazon S3 (free, open)
import boto3, warcio, trafilatura

s3 = boto3.client('s3')
# No auth required. S3 bucket is public.
obj = s3.get_object(Bucket='commoncrawl', Key='crawl-data/CC-MAIN-2021-04/...')

for record in warcio.ArchiveIterator(obj['Body']):
    raw_html = record.content_stream().read()
    text = trafilatura.extract(raw_html)     # strip HTML, extract text
    if text and len(text) > 100:
        save_to_corpus(text)                 # that's it. that's the heist.

# Common Crawl is free on AWS Open Datasets. No auth. No cost to read.
import boto3, io
from warcio.archiveiterator import ArchiveIterator

s3 = boto3.client('s3', region_name='us-east-1')

def stream_warc(crawl_id, segment, file_path):
    key = f'crawl-data/{crawl_id}/{segment}/{file_path}'
    response = s3.get_object(Bucket='commoncrawl', Key=key)
    return ArchiveIterator(response['Body'])

import trafilatura
from langdetect import detect
import fasttext

lang_model = fasttext.load_model('lid.176.bin')  # language ID

def extract(record):
    html = record.content_stream().read().decode('utf-8', errors='ignore')
    text = trafilatura.extract(html, include_comments=False)
    if not text or len(text) < 100:
        return None
    lang, score = lang_model.predict(text[:500])
    if lang[0] != '__label__en' or score[0] < 0.65:
        return None        # English only — other labs do multilingual
    return text

from datasketch import MinHash, MinHashLSH

lsh = MinHashLSH(threshold=0.8, num_perm=128)  # 80% similarity = duplicate

def get_minhash(text):
    m = MinHash(num_perm=128)
    for word in text.lower().split():
        m.update(word.encode('utf8'))
    return m

# Check against corpus — if too similar to existing doc, skip it
def is_duplicate(doc_id, text):
    m = get_minhash(text)
    result = lsh.query(m)
    if result:
        return True   # already have this content
    lsh.insert(doc_id, m)
    return False

# Meta CCNet: train KenLM language model on Wikipedia, score all docs
# High perplexity = low quality. Keep middle band (not too perfect, not garbage)
import kenlm

lm = kenlm.Model('wiki_en.arpa')

def perplexity(text):
    words = text.split()
    log_prob = lm.score(' '.join(words[:200]))
    return 10 ** (-log_prob / len(words))

# Docs above threshold cut. Below threshold cut. Goldilocks band kept.
def quality_filter(text, lo=10, hi=1000):
    p = perplexity(text)
    return lo < p < hi

import apache_beam as beam

with beam.Pipeline(options=PipelineOptions(
    runner='DataflowRunner',       # Google Cloud Dataflow
    project='ai-lab-training',
    region='us-central1',
    machine_type='n1-highmem-64',   # 64 cores, 416GB RAM
    num_workers=500,               # 500 machines running in parallel
)) as p:
    (p
     | 'Read' >> beam.io.ReadFromText('gs://commoncrawl-warc/*.gz')
     | 'Extract' >> beam.Map(extract)
     | 'Filter' >> beam.Filter(quality_filter)
     | 'Dedup' >> beam.transforms.util.Deduplicate()
     | 'Write' >> beam.io.WriteToParquet('gs://training-corpus/'))

import requests
from bs4 import BeautifulSoup

def fetch_from_libgen(isbn):
    # Search libgen — no auth, open HTTP
    search = requests.get(
        f'{LIBGEN_MIRROR}/search.php?req={isbn}&column=identifier')
    soup = BeautifulSoup(search.text, 'html.parser')

    download_link = soup.find('a', string='[1]')['href']
    book_page = requests.get(download_link)
    direct_url = BeautifulSoup(book_page.text, 'html.parser')\
                 .find('a', string='GET')['href']

    return requests.get(direct_url).content  # the book. plaintext. done.

# Scale to 4 million books with asyncio + ThreadPoolExecutor
# Meta chose this over BitTorrent: less traceable. no .torrent metadata.