From fe44bb5b04aa09473311d90057a4717168d74872 Mon Sep 17 00:00:00 2001 From: Harshaneel Gokhale Date: Thu, 9 Jul 2026 18:11:49 -0700 Subject: [PATCH 1/4] CI: skill benchmark gate on PRs (Gemini executor, Binoculars scorer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On PRs touching humanize/SKILL.md: humanize 25 fixed AI-flavored inputs (the documented benchmark registers) with the PR's skill, score outputs and raw inputs with pinned official Binoculars (TinyLlama pair), and enforce a baseline gate: mean lift over the raw inputs and a cap on outputs scoring below their own raw input (thresholds in .github/benchmark/baseline.json). Results are upserted as a marker-keyed PR comment with per-input collapsible inspection (original vs humanized text, score movement, timing), auto-fitted to GitHub's comment limit. Executor: Gemini free tier (gemini-flash-lite-latest rolling alias) via the OpenAI-compatible endpoint. The GEMINI_API_KEY secret is scoped to the `benchmark` environment so it is only usable by approved runs of this workflow. Fork PRs run under pull_request_target: workflow and all scripts execute from the base branch and only the fork's SKILL.md is fetched, as prompt data — fork code never executes next to the secret. Same-repo PRs use the plain pull_request trigger; conditions are mutually exclusive. Resilience: humanize and score are separate jobs with an artifact handoff so "Re-run failed jobs" repeats only the failed half; the executor checkpoints per item and re-run attempts seed from the prior attempt's partial artifact; patient exponential backoff (20s-300s, 6 attempts) absorbs free-tier 429/503 throttling; deps and the Binoculars source are version/SHA-pinned. Co-Authored-By: Claude Fable 5 --- .github/benchmark/baseline.json | 5 + .github/benchmark/humanize_batch.py | 123 +++++++++++++++++ .github/benchmark/inputs.json | 127 ++++++++++++++++++ .github/benchmark/score_and_report.py | 171 ++++++++++++++++++++++++ .github/workflows/benchmark.yml | 183 ++++++++++++++++++++++++++ 5 files changed, 609 insertions(+) create mode 100644 .github/benchmark/baseline.json create mode 100644 .github/benchmark/humanize_batch.py create mode 100644 .github/benchmark/inputs.json create mode 100644 .github/benchmark/score_and_report.py create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/benchmark/baseline.json b/.github/benchmark/baseline.json new file mode 100644 index 0000000..5f72c2d --- /dev/null +++ b/.github/benchmark/baseline.json @@ -0,0 +1,5 @@ +{ + "_comment": "Pass criteria for the skill benchmark. Scores are official-Binoculars (TinyLlama pair), higher = more human. The raw-input baseline is scored fresh every run; thresholds are lifts relative to it. Calibrated 2026-07-09 to the Gemma 3 12B executor, which achieved +0.029 lift with 7 outputs below raw on the then-current skill; thresholds sit just under that so the gate flags regressions, not executor weakness. Tune deliberately, with a PR.", + "min_mean_lift_over_raw": 0.02, + "max_outputs_below_own_raw_input": 8 +} \ No newline at end of file diff --git a/.github/benchmark/humanize_batch.py b/.github/benchmark/humanize_batch.py new file mode 100644 index 0000000..f4044fe --- /dev/null +++ b/.github/benchmark/humanize_batch.py @@ -0,0 +1,123 @@ +"""Humanize the benchmark inputs through a given SKILL.md via an OpenAI-compatible +endpoint (default: a local localaik container, https://github.com/harshaneel/localaik). + +One call per input: the skill text is the system prompt, the input text is the user +message. Keyless and offline-capable. The executor is a small local model, so absolute +scores are not comparable to agentic or frontier-API runs — only to other runs of this +script with the same executor. +""" + +import argparse +import json +import os +import sys +import time + +from openai import OpenAI + +PROMPT = ( + "Apply the humanize skill in your system prompt to the following {register} text. " + "Follow the full protocol: all levers, the pre-output gate, the self-check, and the " + "Signal I audit pass. Output ONLY what the skill instructs you to output.\n\n{text}" +) + +SYSTEM = ( + "You have the following skill loaded. Apply it exactly as written when the user " + "asks you to humanize text.\n\n\n{skill}\n" +) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--skill", required=True, help="Path to SKILL.md") + p.add_argument("--inputs", help="Required unless --warmup") + p.add_argument("--output", help="Required unless --warmup") + p.add_argument("--model", default="gemma-3-4b-it") + p.add_argument("--base-url", default=os.environ.get("LOCALAIK_BASE_URL", "http://localhost:8090/v1")) + p.add_argument("--warmup", action="store_true", + help="Send one tiny request with the skill system prompt so llama.cpp " + "caches the ~9k-token prefix, then exit. Later requests reuse it.") + args = p.parse_args() + + with open(args.skill) as f: + system = SYSTEM.format(skill=f.read()) + + if args.warmup: + client = OpenAI(base_url=args.base_url, api_key="test", timeout=1800.0, max_retries=0) + t0 = time.time() + client.chat.completions.create( + model=args.model, + max_tokens=4, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": "Reply with OK."}, + ], + ) + print(f"Warmup done in {time.time() - t0:.1f}s (skill prefix now in llama.cpp cache)", + file=sys.stderr) + return + + if not args.inputs or not args.output: + p.error("--inputs and --output are required unless --warmup") + with open(args.inputs) as f: + inputs = json.load(f) + + # API key comes from the LLM_API_KEY env var (Gemini key in CI; any placeholder for + # a local keyless server like localaik). Generous timeout covers slow local executors. + api_key = os.environ.get("LLM_API_KEY", "test") + client = OpenAI(base_url=args.base_url, api_key=api_key, timeout=1200.0, max_retries=0) + + # Resume support: a rerun after a crash skips ids already in the output file. + results = [] + done_ids = set() + if os.path.exists(args.output): + with open(args.output) as f: + results = json.load(f) + done_ids = {r["id"] for r in results} + print(f"Resuming: {len(done_ids)} outputs already present", file=sys.stderr) + + for entry in inputs: + if entry["id"] in done_ids: + continue + prompt = PROMPT.format(register=entry["register"], text=entry["text"]) + out = None + t0 = time.time() + # Free-tier capacity throttling (503 "high demand") and rate limits (429) + # clear in minutes, not seconds: exponential backoff, patient tail. + attempts, waits = 6, (20, 40, 80, 160, 300) + for attempt in range(attempts): + try: + resp = client.chat.completions.create( + model=args.model, + # Outputs are ~150-250 tokens; a tight cap keeps decode time down. + max_tokens=800, + temperature=0.7, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": prompt}, + ], + ) + out = (resp.choices[0].message.content or "").strip() + if out: + break + raise RuntimeError("empty completion") + except Exception as e: # noqa: BLE001 - retry any transport/server hiccup + if attempt == attempts - 1: + raise + wait = waits[attempt] + print(f" id={entry['id']} attempt {attempt + 1} failed ({e}); retrying in {wait}s", + file=sys.stderr) + time.sleep(wait) + results.append({"id": entry["id"], "register": entry["register"], "humanized_text": out, + "seconds": round(time.time() - t0, 1)}) + # Checkpoint after every item so a crash never loses the completed portion. + with open(args.output, "w") as f: + json.dump(results, f, indent=1) + print(f" id={entry['id']:>2} done in {time.time() - t0:5.1f}s " + f"({len(out.split())} words) [{entry['register']}]", file=sys.stderr) + + print(f"Wrote {len(results)} outputs to {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/.github/benchmark/inputs.json b/.github/benchmark/inputs.json new file mode 100644 index 0000000..4f50984 --- /dev/null +++ b/.github/benchmark/inputs.json @@ -0,0 +1,127 @@ +[ + { + "id": 1, + "register": "Tech blog post", + "text": "When we set out to rebuild our authentication service, it is important to note that we didn't take the decision lightly; the existing system, built in 2019, had served us reasonably well, but it was starting to show its age. We decided to leverage a more robust, comprehensive framework that would streamline the login flow for our 2.3 million monthly users. Furthermore, the migration involved three pivotal phases: auditing the legacy codebase, designing the new schema, and rolling out changes incrementally to avoid downtime. Moreover, we delved into the tradeoffs between speed, security, and maintainability, weighing each carefully. The new system, powered by a service we internally call Aegis, reduced average login latency from 420ms to 180ms. It is also worth noting that this project required close collaboration between our platform team, our security team, and our infrastructure team. In the end, the results speak for themselves: fewer support tickets, faster onboarding, and a more resilient foundation for whatever comes next. This was, without question, a pivotal moment for our engineering culture." + }, + { + "id": 2, + "register": "Engineering postmortem", + "text": "On March 14th, at approximately 2:47 AM UTC, our payments service experienced a significant outage that impacted roughly 18% of transactions for a duration of 47 minutes; it is important to note that this was not a singular failure but rather the result of three compounding issues. Furthermore, a routine deployment to our database cluster, an unexpected spike in traffic from a marketing campaign, and a misconfigured rate limiter combined to create a perfect storm. Moreover, our on-call engineer, working with limited visibility, had to delve into logs across five separate services before identifying the root cause. We leveraged our robust incident response playbook to restore service, though the process revealed gaps in our comprehensive monitoring setup. In the interest of transparency, it is worth noting that this incident affected approximately 12,000 customers, some of whom experienced failed checkouts. Going forward, we plan to streamline our alerting thresholds, invest in better observability tooling, and conduct quarterly fire drills. This was, undoubtedly, a pivotal learning experience for our team, and we remain committed to preventing similar incidents in the future." + }, + { + "id": 3, + "register": "Product launch announcement", + "text": "Today, we are thrilled to announce the launch of Pulse 3.0, a comprehensive reimagining of our flagship analytics platform that has been two years in the making. It is important to note that this release represents far more than an incremental update; rather, it is a pivotal step forward for the 40,000 businesses that rely on us daily. Furthermore, Pulse 3.0 leverages a robust new data engine capable of processing 10 million events per second, a threefold improvement over our previous architecture. Moreover, we've delved deeply into customer feedback, streamlining the onboarding experience, the dashboard interface, and the reporting workflow. Whether you are a solo founder, a growing startup, or an established enterprise, Pulse 3.0 is designed to meet you where you are. The new pricing starts at $49 per month, with an enterprise tier available for larger teams. We believe this launch will fundamentally transform how our customers understand their data, make decisions, and grow their businesses. It is also worth noting that early beta users, including a well-known retailer, Fenwick & Co., reported a 25% reduction in reporting time." + }, + { + "id": 4, + "register": "Academic abstract", + "text": "This study seeks to delve into the pivotal role of microbial diversity in soil carbon sequestration, an area that, despite considerable research, remains incompletely understood. It is important to note that prior work has often relied on limited sample sizes; our comprehensive analysis, by contrast, draws on 1,240 soil samples collected across 14 distinct biomes between 2021 and 2023. Furthermore, we leverage a robust statistical model to quantify the relationship between bacterial community composition, fungal biomass, and long-term carbon retention. Moreover, our findings suggest that a streamlined framework for classifying soil microbiomes could meaningfully improve predictive accuracy in climate models. The results indicate three key patterns: increased fungal-to-bacterial ratios correlate with higher carbon stability, nitrogen availability moderates this relationship, and regional climate exerts a significant but variable influence. It is also worth noting that these findings have pivotal implications for land management policy, agricultural practice, and climate mitigation strategy more broadly. We conclude that further research is warranted to fully delve into the mechanistic underpinnings of these observed correlations, particularly across underrepresented tropical biomes." + }, + { + "id": 5, + "register": "Business email", + "text": "Hi Sarah, I hope this email finds you well. I wanted to follow up regarding our conversation last Tuesday about the Q3 vendor contract renewal; it is important to note that the deadline is fast approaching, with the current agreement expiring on August 15th. Furthermore, I've delved into the proposed terms and believe we have an opportunity to leverage our five-year relationship with Meridian Supply to negotiate a more favorable rate. Moreover, their comprehensive service package now includes expedited shipping, dedicated account management, and quarterly performance reviews, which could streamline our procurement process significantly. It is also worth noting that our current spend with them totals approximately $340,000 annually, so even a modest 5% reduction would be meaningful. I'd love to set up a call this week, perhaps Thursday afternoon, to discuss next steps and align on our negotiation strategy. In the meantime, please let me know if you have any questions or concerns. This renewal is, in many ways, a pivotal decision for our budget planning heading into next year. Looking forward to hearing from you. Best regards, James" + }, + { + "id": 6, + "register": "Internal Slack update", + "text": "Hey team, quick update on the migration project as we head into the weekend; it is important to note that we've made significant progress since our last sync on Monday. Furthermore, the backend team has successfully delved into the legacy API endpoints and migrated 32 of the 47 total routes to our new, more robust infrastructure. Moreover, we've leveraged the comprehensive test suite that QA put together last sprint, which has helped us catch a handful of edge cases early. There are three things I want to flag: first, the staging environment is currently stable; second, we're still streamlining the authentication layer, which may take a few more days; and third, we'll need everyone's help testing on Monday morning. It is also worth noting that this migration represents a pivotal shift in how we handle data across services, so thorough testing really matters here. Great work so far, everyone, this has been a genuinely collaborative effort across engineering, product, and design. Let's keep the momentum going into next week. Ping me directly if you run into any blockers or have questions." + }, + { + "id": 7, + "register": "LinkedIn post", + "text": "I'm thrilled to share that I've officially stepped into a new role as Director of Product at Northwind Analytics, a company I've admired for quite some time. It is important to note that this journey hasn't been linear; over the past eight years, I've delved into roles spanning engineering, design, and go-to-market strategy, each one teaching me something pivotal. Furthermore, I've learned to leverage cross-functional collaboration, robust data-driven decision-making, and comprehensive stakeholder alignment as the true pillars of great product work. Moreover, I want to thank my incredible mentors, my supportive family, and my former colleagues at Bright Path Solutions, where I spent the last four years building something I'm genuinely proud of. To anyone early in their career, my advice is simple: stay curious, stay humble, and never stop learning. It is also worth noting that this new chapter comes with plenty of unknowns, but I couldn't be more excited to streamline our roadmap and help Northwind scale to new heights. Here's to new beginnings, new challenges, and new opportunities ahead. Thank you all for the outpouring of support." + }, + { + "id": 8, + "register": "Cover letter", + "text": "Dear Hiring Manager, I am writing to express my sincere interest in the Senior Software Engineer position at Clearwater Systems, as advertised on your careers page. It is important to note that my background spans five years of experience across backend development, cloud infrastructure, and team leadership, and I believe this comprehensive skill set aligns well with your needs. Furthermore, in my current role at Delacroix Technologies, I have leveraged robust engineering practices to streamline our deployment pipeline, reducing release time from three days to just six hours. Moreover, I delved deeply into our microservices architecture, identifying and resolving three pivotal bottlenecks that had long plagued our system's performance. I am particularly drawn to Clearwater's mission of making financial tools more accessible, transparent, and equitable. It is also worth noting that I thrive in collaborative, fast-paced environments where I can leverage both technical expertise and interpersonal skills. I would welcome the opportunity to discuss how my experience could contribute meaningfully to your team's continued success. Thank you for considering my application; I look forward to the possibility of speaking further. Sincerely, Alex Rivera" + }, + { + "id": 9, + "register": "Marketing copy", + "text": "Introducing BrewMind, the coffee maker that doesn't just brew coffee, it delves into the science of flavor extraction to deliver a truly exceptional cup, every single time. It is important to note that BrewMind isn't just another appliance; it's a comprehensive, robust solution engineered to streamline your entire morning routine. Furthermore, with three customizable brew modes, a 12-cup capacity, and a sleek stainless-steel finish, BrewMind fits seamlessly into any kitchen, any lifestyle, any budget. Moreover, our proprietary Precision Bloom technology leverages a four-stage saturation process to unlock notes that ordinary machines simply can't reach. Whether you're a casual drinker, a self-proclaimed coffee snob, or somewhere in between, BrewMind was designed with you in mind. It is also worth noting that over 50,000 units have already sold since launch, with a 4.8-star average rating across more than 3,200 reviews. At just $89.99, BrewMind represents a pivotal upgrade for anyone serious about their morning cup. Don't settle for mediocre coffee any longer; streamline your routine, elevate your mornings, and experience the difference for yourself today." + }, + { + "id": 10, + "register": "Press release", + "text": "FOR IMMEDIATE RELEASE. San Francisco, CA, June 3, 2026. Meridian Health Technologies today announced the launch of its comprehensive telehealth platform, CarePath, designed to streamline access to care for patients across underserved rural communities. It is important to note that this launch follows an 18-month development period and a successful pilot program involving 22 clinics across four states. Furthermore, CarePath leverages a robust, HIPAA-compliant infrastructure to connect patients with over 1,500 licensed providers nationwide. Moreover, company leadership delved into extensive user research, incorporating feedback from more than 6,000 patients during the beta phase. \"This launch represents a pivotal moment for healthcare accessibility,\" said CEO Danielle Ford. \"We believe CarePath will fundamentally change how rural patients experience care.\" It is also worth noting that the platform will be available starting July 15th, with pricing plans beginning at $29 per month for individuals. The company also announced a partnership with Redwood Regional Hospital System to further expand its comprehensive network of care providers. Meridian Health Technologies is headquartered in San Francisco and currently employs 340 people across three offices." + }, + { + "id": 11, + "register": "Investor update", + "text": "Dear investors, I hope this update finds you well as we close out Q2. It is important to note that this quarter has been a pivotal one for the company, marked by significant, measurable progress across the board. Furthermore, revenue grew to $1.4 million, a 22% increase quarter-over-quarter, while our customer base expanded to 3,800 active accounts. Moreover, we've delved deeply into our unit economics, and I'm pleased to report that our gross margin has improved to 68%, up from 61% last quarter. We leveraged a more robust go-to-market strategy this quarter, streamlining our sales process and reducing our average deal cycle from 45 days to 31 days. It is also worth noting that we closed a comprehensive Series A round of $8 million, led by Fenway Ventures, with participation from three existing investors. On the challenges side, hiring has been slower than anticipated; we've filled only 4 of 9 open engineering roles. Looking ahead, our priorities remain threefold: scaling the sales team, deepening our enterprise partnerships, and continuing to streamline our product roadmap. Thank you, as always, for your continued support and trust." + }, + { + "id": 12, + "register": "Job posting", + "text": "Northgate Robotics is seeking a Senior Backend Engineer to join our growing team of 45 employees in Austin, Texas. It is important to note that this role offers a unique opportunity to delve into cutting-edge robotics infrastructure, working alongside a comprehensive team of engineers, designers, and product managers. Furthermore, the ideal candidate will leverage at least five years of experience with distributed systems, robust API design, and cloud infrastructure, ideally with AWS or GCP. Moreover, you'll be responsible for streamlining our data pipelines, which currently process over 2 million sensor readings per day across our fleet of autonomous units. This is, without question, a pivotal role for our engineering organization as we scale toward our next funding round. Responsibilities include: designing scalable backend systems, mentoring junior engineers, and collaborating cross-functionally with hardware teams. It is also worth noting that we offer a comprehensive benefits package, including full health coverage, a $2,000 annual learning stipend, and unlimited PTO. Salary range is $150,000 to $190,000, depending on experience. If you're passionate about robotics and eager to leverage your skills in a fast-paced environment, we'd love to hear from you." + }, + { + "id": 13, + "register": "Customer support reply", + "text": "Hi Marcus, thank you for reaching out, and I sincerely apologize for the inconvenience you've experienced with your recent order, #48213. It is important to note that our team has thoroughly delved into your account history to understand exactly what went wrong. Furthermore, it appears that a warehouse processing error, a shipping label mismatch, and a delayed carrier pickup combined to cause the three-day delay you encountered. Moreover, I want to assure you that we take this matter seriously and are leveraging our internal quality assurance process to prevent similar issues going forward. As a gesture of goodwill, we'd like to offer you a full refund of $34.99, as well as a 20% discount code for your next purchase. It is also worth noting that our shipping partner has since implemented a more robust tracking system to streamline future deliveries. We genuinely value your business and hope this comprehensive resolution meets your expectations. Please don't hesitate to reach out if you have any further questions or concerns; we're here to help every step of the way. Thank you again for your patience and understanding, Marcus. Warm regards, Priya from Customer Support" + }, + { + "id": 14, + "register": "Recipe intro", + "text": "There's something truly special about a comprehensive, home-cooked lasagna, and it is important to note that this particular recipe has been in my family for over 30 years. Furthermore, I've delved into countless variations over the years, tweaking the sauce, the cheese blend, and the pasta layers until I finally landed on what I consider the perfect, robust balance of flavors. Moreover, this recipe leverages a rich, slow-simmered marinara made with San Marzano tomatoes, fresh basil, and a touch of red wine, which streamlines the cooking process without sacrificing depth of flavor. Whether you're cooking for a small family dinner or a larger gathering, this dish serves eight generously and reheats beautifully the next day. It is also worth noting that the total preparation time is approximately 90 minutes, including a 45-minute bake. I first made this dish for my grandmother's 80th birthday in 2014, and it's remained a pivotal part of our holiday traditions ever since. Grab your favorite baking dish, roll up your sleeves, and let's delve into this comforting, three-layered classic together." + }, + { + "id": 15, + "register": "Travel writing", + "text": "Lisbon is, without question, a city that rewards those willing to delve into its winding streets, historic trams, and vibrant, ever-evolving neighborhoods. It is important to note that the city's charm lies not in any single landmark but in the comprehensive experience of wandering through Alfama, Bairro Alto, and Belem, each offering something pivotal to the overall journey. Furthermore, I spent five days exploring the city in April of 2025, and I leveraged a combination of walking tours, local recommendations, and spontaneous detours to get a truly robust sense of the place. Moreover, the pasteis de nata from Pasteis de Belem, a bakery operating since 1837, remain some of the finest I've ever tasted. Whether you're drawn to the panoramic viewpoints, the centuries-old architecture, or the lively fado music echoing through narrow alleyways, Lisbon offers a streamlined yet deeply layered introduction to Portuguese culture. It is also worth noting that a single metro ticket costs just 1.65 euros, making the city remarkably accessible for budget travelers. This was, for me, a genuinely pivotal trip, one that reshaped how I think about slow, intentional travel." + }, + { + "id": 16, + "register": "Restaurant review", + "text": "Tucked away on a quiet corner in the Mission District, Casa Verde has quickly become, in my opinion, a pivotal addition to San Francisco's ever-evolving culinary scene. It is important to note that the restaurant, which opened in November 2024, leverages a comprehensive, farm-to-table philosophy that shows in every dish. Furthermore, I delved into the tasting menu, priced at $95 per person, which included three standout courses: a robust roasted beet salad, a delicately seared branzino, and a rich, comprehensive short rib finished with a red wine reduction. Moreover, the service was attentive without being intrusive, the wine list was thoughtfully curated, and the ambiance struck a streamlined balance between rustic and refined. It is also worth noting that the restaurant seats only 40 guests, so reservations are highly recommended, particularly on weekends. While the portions leaned smaller than expected given the price point, the quality more than compensated for it. Overall, Casa Verde represents a genuinely pivotal moment for the neighborhood, and I'll certainly be delving back into their ever-changing seasonal menu very soon." + }, + { + "id": 17, + "register": "Book review", + "text": "In her latest novel, The Weight of Silence, author Clara Bennington delves into themes of grief, memory, and reconciliation with a comprehensive, deeply affecting narrative structure. It is important to note that the novel, published in March 2025 by Harrow & Finch Press, spans nearly 420 pages, yet it rarely feels overlong. Furthermore, Bennington leverages a robust, multi-generational storyline to explore three pivotal relationships: a mother and daughter, two estranged sisters, and a widow confronting her late husband's secrets. Moreover, the prose is streamlined without sacrificing emotional depth, moving fluidly between past and present with remarkable clarity. It is also worth noting that this is Bennington's fourth novel, following her critically acclaimed debut in 2019, and it arguably represents her most comprehensive work to date. While the pacing occasionally lags in the middle third, the payoff in the final chapters is genuinely pivotal, tying together threads that had, until then, felt loosely connected. I would recommend this book to readers who enjoy delving into slow-burning, character-driven family dramas with real emotional weight." + }, + { + "id": 18, + "register": "Personal essay", + "text": "It is important to note that grief doesn't arrive on any predictable schedule; it simply shows up, uninvited, and forces you to delve into feelings you didn't know you were avoiding. Furthermore, when my father passed away in the fall of 2021, I found myself leveraging routines, small comprehensive rituals like morning coffee and evening walks, just to get through each day. Moreover, I spent countless hours delving into old photographs, letters, and voicemails, searching for some robust sense of closure that never quite arrived. There were three things that helped me most during that period: therapy, time spent with close friends, and, oddly enough, learning to cook the recipes my father loved. It is also worth noting that healing isn't a streamlined, linear process; some days felt pivotal and transformative, while others felt like standing still. I've come to believe that grief, in its own strange way, is a comprehensive teacher, one that leverages loss to reshape how we understand love, memory, and time itself. This essay is, in many ways, my attempt to make sense of something that still doesn't fully make sense." + }, + { + "id": 19, + "register": "Privacy policy section", + "text": "This section is intended to provide a comprehensive overview of how we collect, use, and store your personal information; it is important to note that we take your privacy seriously and have implemented robust safeguards accordingly. Furthermore, we may collect data including your name, email address, IP address, and browsing behavior in order to streamline your overall experience on our platform. Moreover, we leverage this information for three primary purposes: improving our services, personalizing your experience, and ensuring compliance with applicable legal obligations. It is also worth noting that we retain personal data for a period of 24 months following account closure, unless a longer retention period is required by law. Furthermore, we may share your data with trusted third-party vendors, including our comprehensive analytics provider, Insight Metrics Inc., though we never sell your personal information to advertisers. This policy was last updated on May 1, 2026, and represents a pivotal step in our ongoing commitment to transparency. Should you have any questions regarding this policy, or wish to delve further into your specific data rights, please contact our privacy team directly at privacy@example.com." + }, + { + "id": 20, + "register": "Tutorial intro", + "text": "In this tutorial, we're going to delve into the process of building a comprehensive REST API using Node.js and Express, a pivotal skill for any aspiring backend developer. It is important to note that by the end of this guide, you'll have a fully functional, robust API capable of handling three core operations: creating, reading, and updating records in a PostgreSQL database. Furthermore, we'll leverage popular libraries such as Sequelize and dotenv to streamline our configuration and database management. Moreover, this tutorial assumes you have Node.js version 18 or higher installed, along with a basic, comprehensive understanding of JavaScript fundamentals. It is also worth noting that the entire project should take approximately two hours to complete, assuming no major setup issues arise. We'll begin by initializing our project, then move on to configuring our robust middleware stack, and finally, we'll delve into error handling and validation. By streamlining each step into clear, manageable sections, this tutorial aims to make what can otherwise feel like a daunting, pivotal task far more approachable. Let's get started by opening our terminal and running npm init." + }, + { + "id": 21, + "register": "Comparison article", + "text": "When it comes to choosing between Notion and Obsidian, it is important to note that both platforms offer a comprehensive suite of features, though they leverage fundamentally different underlying philosophies. Furthermore, Notion, which reported over 30 million users as of 2023, provides a robust, all-in-one workspace that streamlines documents, databases, and collaboration into a single, unified platform. Moreover, Obsidian, by contrast, delves into a more pivotal, file-based approach, storing everything locally in markdown files, which many privacy-conscious users find appealing. There are three key differences worth considering: pricing, with Obsidian offering a free tier and Notion charging $10 per user monthly for its team plan; collaboration features, where Notion clearly leads; and customization, where Obsidian's plugin ecosystem, comprising over 1,000 community plugins, offers unmatched flexibility. It is also worth noting that Notion tends to perform better for teams, while Obsidian remains a comprehensive favorite among individual writers and researchers. Ultimately, the choice comes down to whether you prioritize a streamlined, collaborative experience or a robust, locally-stored, pivotal system built for personal knowledge management." + }, + { + "id": 22, + "register": "Roadmap update", + "text": "As we look ahead to the second half of 2026, it is important to note that our product roadmap reflects a comprehensive, pivotal shift in priorities based on extensive customer feedback. Furthermore, we've delved into over 400 support tickets and 60 customer interviews to identify the three most requested features: real-time collaboration, a robust mobile app, and streamlined third-party integrations. Moreover, we plan to leverage our existing infrastructure to ship the mobile app by Q3, with a public beta launching in September. It is also worth noting that real-time collaboration, a genuinely pivotal feature for our enterprise customers, will require a comprehensive rebuild of our underlying data layer, currently estimated at 12 weeks of engineering effort. We're also streamlining our integration marketplace, which currently supports 18 third-party tools, with plans to expand to 35 by year's end. Additionally, we'll continue to leverage user feedback through our robust beta testing program, which currently includes 250 active participants. This roadmap represents a comprehensive, pivotal step forward for our platform, and we remain committed to delivering meaningful, measurable value to our growing user base." + }, + { + "id": 23, + "register": "Conference abstract", + "text": "This talk will delve into the pivotal challenges of scaling distributed systems, drawing on comprehensive, real-world case studies from three separate production environments. It is important to note that as organizations increasingly leverage microservices architectures, the need for robust, fault-tolerant infrastructure becomes ever more pressing. Furthermore, this session will explore how our team, over the course of 18 months, streamlined our deployment pipeline to handle over 500 million requests per day. Moreover, attendees will gain a comprehensive understanding of three key strategies: implementing robust circuit breakers, leveraging comprehensive observability tooling, and streamlining incident response protocols. It is also worth noting that this talk builds on research first presented at a smaller internal conference in 2024, expanding significantly on those pivotal early findings. Attendees will leave with actionable, practical takeaways they can immediately leverage within their own organizations, regardless of scale. This session is intended for engineers, architects, and technical leaders alike, and will run for approximately 45 minutes, followed by a comprehensive 15-minute Q&A session delving further into audience questions." + }, + { + "id": 24, + "register": "README intro", + "text": "Welcome to Streamline, a comprehensive, open-source toolkit designed to help developers delve into complex data pipelines with minimal friction and maximum robustness. It is important to note that this project was first launched in January 2023 and has since grown to over 4,200 stars on GitHub, with contributions from more than 85 developers worldwide. Furthermore, Streamline leverages a robust, modular architecture that allows you to plug in custom transformations, validators, and connectors without needing to modify the comprehensive core codebase. Moreover, this README will guide you through three essential steps: installation, configuration, and running your very first pipeline. It is also worth noting that Streamline currently supports Python 3.9 and above, and requires a minimum of 4GB of RAM for most standard workloads. To get started, simply run pip install streamline-toolkit, and you'll be up and running within minutes. We believe this project represents a pivotal step forward for teams looking to streamline their data engineering workflows without sacrificing flexibility or control. Contributions, issues, and feature requests are always welcome; please delve into our CONTRIBUTING.md file for more comprehensive details." + }, + { + "id": 25, + "register": "Career advice", + "text": "It is important to note that navigating a career transition, especially later in life, requires you to delve into your priorities with real, comprehensive honesty. Furthermore, I made the pivotal decision to leave my robust, comfortable job at a Fortune 500 company in 2020 to pursue something more meaningful, and it wasn't easy. Moreover, I leveraged my network, my savings, and roughly 18 months of careful planning to make the transition as smooth as possible. There are three pieces of advice I'd offer anyone considering a similar leap: build a comprehensive financial cushion before you leap, leverage your existing network rather than starting from scratch, and streamline your job search by focusing on quality applications over quantity. It is also worth noting that I sent out only 12 applications during my transition, compared to the hundreds many career coaches recommend, and I landed a role I genuinely love. This pivotal experience taught me that a robust, comprehensive plan matters far more than sheer volume or speed. Ultimately, careers are rarely linear, and learning to delve into uncertainty, rather than avoid it, may be the most valuable skill of all." + } +] diff --git a/.github/benchmark/score_and_report.py b/.github/benchmark/score_and_report.py new file mode 100644 index 0000000..c47852d --- /dev/null +++ b/.github/benchmark/score_and_report.py @@ -0,0 +1,171 @@ +"""Score raw inputs and PR-skill outputs with official Binoculars (TinyLlama pair), +check the outputs against the committed baseline thresholds, and emit a markdown +report for a PR comment. + +Score direction: higher = more human-like. The raw inputs are scored fresh in every +run so the baseline shares the run's environment; thresholds in baseline.json are +lifts relative to that, not absolute scores. +""" + +import argparse +import json +import sys +from statistics import mean, median + +MARKER = "" + + +def strip_meta_note(text: str) -> str: + return text.split("\n\n[Note:")[0].split("[Note:")[0].strip() + + +def load(path, text_key_candidates=("humanized_text", "text")): + with open(path) as f: + entries = json.load(f) + out = {} + for e in entries: + text = next(e[k] for k in text_key_candidates if k in e) + out[e["id"]] = {"register": e["register"], "text": strip_meta_note(text), + "seconds": e.get("seconds")} + return out + + +def item_block(i, raw, out, scores, cap): + """One collapsible block: score summary line, original and humanized text inside.""" + def trunc(t): + if len(t) <= cap: + return t + return t[:cap] + " … _(truncated; full text in the run artifact)_" + + def quote(t): + return "> " + t.replace("\n", "\n> ") + + ok = "✅" if scores["out"][i] >= scores["raw"][i] else "❌" + words = len(out[i]["text"].split()) + secs = out[i].get("seconds") + timing = f" · {secs:.0f}s" if secs else "" + summary = (f"id {i} · {raw[i]['register']} · raw {scores['raw'][i]:.3f} → " + f"humanized {scores['out'][i]:.3f} {ok} · {words} words{timing}") + return "\n".join([ + f"
{summary}", + "", + "**Original (AI-flavored input):**", + "", + quote(trunc(raw[i]["text"])), + "", + "**Humanized (PR skill):**", + "", + quote(trunc(out[i]["text"])), + "", + "
", + ]) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--inputs", required=True) + p.add_argument("--outputs", required=True) + p.add_argument("--baseline", required=True, help="baseline.json with pass thresholds") + p.add_argument("--report", required=True, help="Markdown report path") + p.add_argument("--json-out", required=True) + p.add_argument("--executor-label", default="a local one-shot executor", + help="Shown in the report, e.g. 'Google Gemini (gemini-2.5-flash)'") + args = p.parse_args() + + with open(args.baseline) as f: + thresholds = json.load(f) + min_lift = thresholds["min_mean_lift_over_raw"] + max_below = thresholds["max_outputs_below_own_raw_input"] + + from binoculars import Binoculars + bino = Binoculars( + observer_name_or_path="TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", + performer_name_or_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + use_bfloat16=False, + ) + print("Binoculars loaded.", file=sys.stderr) + + raw = load(args.inputs) + out = load(args.outputs) + scores = {"raw": {}, "out": {}} + for arm, entries in (("raw", raw), ("out", out)): + for i, e in sorted(entries.items()): + scores[arm][i] = float(bino.compute_score(e["text"])) + print(f" {arm:3s} id={i:>2} score={scores[arm][i]:.4f}", file=sys.stderr) + + ids = sorted(raw) + def agg(arm): + vals = [scores[arm][i] for i in ids] + return {"mean": mean(vals), "median": median(vals), "min": min(vals), "max": max(vals)} + + a_raw, a_out = agg("raw"), agg("out") + lift = a_out["mean"] - a_raw["mean"] + below = [i for i in ids if scores["out"][i] < scores["raw"][i]] + lift_ok = lift >= min_lift + below_ok = len(below) <= max_below + passed = lift_ok and below_ok + + def check(ok): + return "✅" if ok else "❌" + + lines = [ + MARKER, + f"## Humanize skill benchmark — {'PASS ✅' if passed else 'FAIL ❌'}", + "", + f"{len(ids)} fixed AI-flavored inputs humanized with the PR's `humanize/SKILL.md` " + f"(executor: {args.executor_label}), then scored with official Binoculars " + "(TinyLlama pair, higher = more human) against the raw inputs as baseline.", + "", + "| Metric | raw inputs (baseline) | humanized (PR skill) |", + "|---|---|---|", + f"| Mean | {a_raw['mean']:.4f} | **{a_out['mean']:.4f}** |", + f"| Median | {a_raw['median']:.4f} | {a_out['median']:.4f} |", + f"| Min | {a_raw['min']:.4f} | {a_out['min']:.4f} |", + f"| Max | {a_raw['max']:.4f} | {a_out['max']:.4f} |", + "", + "| Check | Threshold | Actual | |", + "|---|---|---|---|", + f"| Mean lift over baseline | ≥ {min_lift:+.3f} | {lift:+.4f} | {check(lift_ok)} |", + f"| Outputs below their own raw input | ≤ {max_below} | {len(below)} | {check(below_ok)} |", + ] + if below: + lines += [ + "", + "
Outputs that scored below their raw input", + "", + "| Register | raw | humanized |", + "|---|---|---|", + ] + lines += [f"| {raw[i]['register']} | {scores['raw'][i]:.3f} | {scores['out'][i]:.3f} |" + for i in below] + lines += ["", "
"] + tail = [ + "", + "_Caveats: single run, no significance testing; one-shot executor, so protocol " + "compliance is weaker than agentic runs; perplexity-class detector only — says " + "nothing about learned classifiers (GPTZero, Grammarly)._", + ] + if "localaik" in args.executor_label.lower(): + tail += ["", + "_Powered by [localaik](https://github.com/harshaneel/localaik) for local testing._"] + + # Per-input inspection blocks, shrunk to fit GitHub's 65,536-char comment limit. + for cap in (1200, 600, 300): + inspection = ["", "### Per-input inspection", ""] + inspection += [item_block(i, raw, out, scores, cap) for i in ids] + report = "\n".join(lines + inspection + tail) + if len(report) < 60000: + break + else: + report = "\n".join(lines + tail) + with open(args.report, "w") as f: + f.write(report) + with open(args.json_out, "w") as f: + json.dump({"pass": passed, "mean_lift": lift, "outputs_below_raw": below, + "aggregates": {"raw": a_raw, "humanized": a_out}, + "per_input": scores, "thresholds": thresholds}, f, indent=1) + print(report) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..8ea69f7 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,183 @@ +name: skill-benchmark + +on: + # Same-repo PRs: workflow and code run from the PR branch (trusted). + pull_request: + paths: + - "humanize/SKILL.md" + - ".github/benchmark/**" + - ".github/workflows/benchmark.yml" + # Fork PRs: workflow and ALL code run from the base branch; only the fork's + # humanize/SKILL.md is fetched, as prompt data. Fork code never executes here. + pull_request_target: + types: [opened, synchronize, reopened, labeled] + paths: + - "humanize/SKILL.md" + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: benchmark-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + BENCH: .github/benchmark + # Rolling alias: tracks Google's current flash-lite model, survives model retirements + # (fixed ids like gemini-2.5-flash 404 for accounts created after retirement, and the + # top flash tier free lane throws 503 "high demand" at peak; lite has capacity headroom). + EXECUTOR_MODEL: gemini-flash-lite-latest + EXECUTOR_BASE_URL: https://generativelanguage.googleapis.com/v1beta/openai/ + +jobs: + # Job 1: LLM inference via the Gemini free tier. The GEMINI_API_KEY secret lives in + # the `benchmark` environment with a required reviewer, so every run that needs it + # waits for maintainer approval and the key is unusable outside this job. + humanize: + # pull_request covers same-repo PRs; pull_request_target covers forks. The two + # conditions are mutually exclusive so a same-repo PR never runs twice. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) + runs-on: ubuntu-latest + environment: benchmark + timeout-minutes: 45 + steps: + # For pull_request this checks out the PR merge ref; for pull_request_target it + # checks out the BASE branch (safe default) — fork code is never checked out. + - uses: actions/checkout@v4 + + - name: Take the fork PR's SKILL.md as data (fork PRs only) + if: github.event_name == 'pull_request_target' + run: | + git fetch origin "pull/${{ github.event.pull_request.number }}/head" + git checkout FETCH_HEAD -- humanize/SKILL.md + echo "Benchmarking fork skill: $(wc -w < humanize/SKILL.md) words" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install client deps + run: pip install --quiet openai + + - name: Seed partial outputs from previous attempt (re-runs only) + if: github.run_attempt != '1' + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: humanized-outputs + path: /tmp + + - name: Humanize inputs with the PR skill + # Resumes past any ids already present in /tmp/outputs.json (seeded above on re-runs). + env: + LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + python $BENCH/humanize_batch.py --skill humanize/SKILL.md \ + --inputs $BENCH/inputs.json --output /tmp/outputs.json \ + --base-url "$EXECUTOR_BASE_URL" --model "$EXECUTOR_MODEL" + + - name: Upload humanized outputs + if: always() + uses: actions/upload-artifact@v4 + with: + name: humanized-outputs + path: /tmp/outputs.json + if-no-files-found: error + overwrite: true + + # Job 2: scoring + report + gate. No secret needed, separate runner, so a scoring + # failure is a cheap "Re-run failed jobs" that reuses the inference artifact. + score: + needs: humanize + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Cache HF models (Binoculars scorer) + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: hf-tinyllama-binoculars-v1 + + - name: Install scorer deps + run: | + pip install --quiet torch==2.12.0 --index-url https://download.pytorch.org/whl/cpu + pip install --quiet transformers==5.9.0 accelerate==1.13.0 numpy==2.4.6 + git clone https://github.com/ahans30/Binoculars.git /tmp/binoculars-src + git -C /tmp/binoculars-src checkout c8ae2f90d50ee696418bc71d8d9e5020e5f9d7b8 + pip install --quiet --no-deps -e /tmp/binoculars-src + + - name: Download humanized outputs + uses: actions/download-artifact@v4 + with: + name: humanized-outputs + path: /tmp + + - name: Score against baseline and build report + run: | + python $BENCH/score_and_report.py \ + --inputs $BENCH/inputs.json \ + --outputs /tmp/outputs.json \ + --baseline $BENCH/baseline.json \ + --executor-label "Google Gemini (\`$EXECUTOR_MODEL\`, free tier, one-shot)" \ + --report /tmp/report.md --json-out /tmp/results.json + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-report + path: | + /tmp/results.json + /tmp/report.md + overwrite: true + + - name: Post or update PR comment + # pull_request_target grants a write token, so fork-PR comments work too. + if: always() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + if (!fs.existsSync('/tmp/report.md')) { + core.warning('No report produced; skipping comment.'); + return; + } + const body = fs.readFileSync('/tmp/report.md', 'utf8'); + const marker = ''; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Enforce baseline + run: | + python -c "import json,sys; r=json.load(open('/tmp/results.json')); sys.exit(0 if r['pass'] else 1)" From 66b5abefd5f419e5e1ad5ec8aff69a311aa7c78c Mon Sep 17 00:00:00 2001 From: Harshaneel Gokhale Date: Thu, 9 Jul 2026 18:28:21 -0700 Subject: [PATCH 2/4] CI benchmark: fold inspection into one dropdown, reading guide, real thresholds - Per-input inspection now nests all item dropdowns inside a single outer dropdown so the comment stays compact - One-line "how to read" note before the score tables; caveats footer removed - Thresholds calibrated to the measured Gemini flash-lite run (+0.076 lift, 2/25 below raw): gate now requires mean lift >= +0.05 and at most 4 outputs below their own raw input Co-Authored-By: Claude Fable 5 --- .github/benchmark/baseline.json | 6 +++--- .github/benchmark/score_and_report.py | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/benchmark/baseline.json b/.github/benchmark/baseline.json index 5f72c2d..4c091a6 100644 --- a/.github/benchmark/baseline.json +++ b/.github/benchmark/baseline.json @@ -1,5 +1,5 @@ { - "_comment": "Pass criteria for the skill benchmark. Scores are official-Binoculars (TinyLlama pair), higher = more human. The raw-input baseline is scored fresh every run; thresholds are lifts relative to it. Calibrated 2026-07-09 to the Gemma 3 12B executor, which achieved +0.029 lift with 7 outputs below raw on the then-current skill; thresholds sit just under that so the gate flags regressions, not executor weakness. Tune deliberately, with a PR.", - "min_mean_lift_over_raw": 0.02, - "max_outputs_below_own_raw_input": 8 + "_comment": "Pass criteria for the skill benchmark. Scores are official-Binoculars (TinyLlama pair), higher = more human. The raw-input baseline is scored fresh every run; thresholds are relative to it. Calibrated 2026-07-10 to the Gemini flash-lite executor, which measured +0.076 mean lift with 2/25 outputs below raw on the then-current skill; thresholds sit just under that so the gate flags skill regressions, not executor noise. Reference: frontier agentic runs measure ~+0.146 with 0 below raw. Tune deliberately, with a PR.", + "min_mean_lift_over_raw": 0.05, + "max_outputs_below_own_raw_input": 4 } \ No newline at end of file diff --git a/.github/benchmark/score_and_report.py b/.github/benchmark/score_and_report.py index c47852d..bea68c5 100644 --- a/.github/benchmark/score_and_report.py +++ b/.github/benchmark/score_and_report.py @@ -116,6 +116,10 @@ def check(ok): f"(executor: {args.executor_label}), then scored with official Binoculars " "(TinyLlama pair, higher = more human) against the raw inputs as baseline.", "", + "**How to read:** each score estimates how human the text reads (higher = more " + "human). A humanized output should score above the raw AI input it came from; the " + "gate checks the average lift and how many outputs fell below their own input.", + "", "| Metric | raw inputs (baseline) | humanized (PR skill) |", "|---|---|---|", f"| Mean | {a_raw['mean']:.4f} | **{a_out['mean']:.4f}** |", @@ -139,20 +143,21 @@ def check(ok): lines += [f"| {raw[i]['register']} | {scores['raw'][i]:.3f} | {scores['out'][i]:.3f} |" for i in below] lines += ["", ""] - tail = [ - "", - "_Caveats: single run, no significance testing; one-shot executor, so protocol " - "compliance is weaker than agentic runs; perplexity-class detector only — says " - "nothing about learned classifiers (GPTZero, Grammarly)._", - ] + tail = [] if "localaik" in args.executor_label.lower(): tail += ["", "_Powered by [localaik](https://github.com/harshaneel/localaik) for local testing._"] - # Per-input inspection blocks, shrunk to fit GitHub's 65,536-char comment limit. + # Per-input inspection: one outer dropdown wrapping the per-item dropdowns, shrunk + # to fit GitHub's 65,536-char comment limit. for cap in (1200, 600, 300): - inspection = ["", "### Per-input inspection", ""] + inspection = [ + "", + f"
Per-input inspection ({len(ids)} items)", + "", + ] inspection += [item_block(i, raw, out, scores, cap) for i in ids] + inspection += ["", "
"] report = "\n".join(lines + inspection + tail) if len(report) < 60000: break From c563ae0e1b0a6281bb1464dd477c9022e1e04a06 Mon Sep 17 00:00:00 2001 From: Harshaneel Gokhale Date: Thu, 9 Jul 2026 18:28:43 -0700 Subject: [PATCH 3/4] CI benchmark: name the verification model in the PR comment The comment now states explicitly that scores come from the official Binoculars detector (Hans et al., ICML 2024) with the TinyLlama-1.1B base + chat model pair. Co-Authored-By: Claude Fable 5 --- .github/benchmark/score_and_report.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/benchmark/score_and_report.py b/.github/benchmark/score_and_report.py index bea68c5..eb82415 100644 --- a/.github/benchmark/score_and_report.py +++ b/.github/benchmark/score_and_report.py @@ -113,8 +113,12 @@ def check(ok): f"## Humanize skill benchmark — {'PASS ✅' if passed else 'FAIL ❌'}", "", f"{len(ids)} fixed AI-flavored inputs humanized with the PR's `humanize/SKILL.md` " - f"(executor: {args.executor_label}), then scored with official Binoculars " - "(TinyLlama pair, higher = more human) against the raw inputs as baseline.", + f"(executor: {args.executor_label}) and scored against the raw inputs as baseline.", + "", + "**Verification model:** scores come from the official " + "[Binoculars](https://github.com/ahans30/Binoculars) zero-shot AI-text detector " + "(Hans et al., ICML 2024) running the `TinyLlama-1.1B` base + `TinyLlama-1.1B-Chat` " + "model pair.", "", "**How to read:** each score estimates how human the text reads (higher = more " "human). A humanized output should score above the raw AI input it came from; the " From 1c07b5f16fda932504a68a538d4496f80de1311f Mon Sep 17 00:00:00 2001 From: Harshaneel Gokhale Date: Thu, 9 Jul 2026 18:30:54 -0700 Subject: [PATCH 4/4] CI benchmark: neutral scorer wording, show resolved Gemini model version - Comment wording: "scoring model"/"scorer" instead of detector-style language - The executor records the concrete model version each response resolved to (rolling aliases hide it), and the comment now shows the distinct resolved names next to the executor label Co-Authored-By: Claude Fable 5 --- .github/benchmark/humanize_batch.py | 3 ++- .github/benchmark/score_and_report.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/benchmark/humanize_batch.py b/.github/benchmark/humanize_batch.py index f4044fe..fe9da85 100644 --- a/.github/benchmark/humanize_batch.py +++ b/.github/benchmark/humanize_batch.py @@ -109,7 +109,8 @@ def main(): file=sys.stderr) time.sleep(wait) results.append({"id": entry["id"], "register": entry["register"], "humanized_text": out, - "seconds": round(time.time() - t0, 1)}) + "seconds": round(time.time() - t0, 1), + "model": getattr(resp, "model", None) or args.model}) # Checkpoint after every item so a crash never loses the completed portion. with open(args.output, "w") as f: json.dump(results, f, indent=1) diff --git a/.github/benchmark/score_and_report.py b/.github/benchmark/score_and_report.py index eb82415..2d3cd9a 100644 --- a/.github/benchmark/score_and_report.py +++ b/.github/benchmark/score_and_report.py @@ -26,7 +26,7 @@ def load(path, text_key_candidates=("humanized_text", "text")): for e in entries: text = next(e[k] for k in text_key_candidates if k in e) out[e["id"]] = {"register": e["register"], "text": strip_meta_note(text), - "seconds": e.get("seconds")} + "seconds": e.get("seconds"), "model": e.get("model")} return out @@ -108,15 +108,21 @@ def agg(arm): def check(ok): return "✅" if ok else "❌" + # Aliases like gemini-flash-lite-latest resolve to a concrete version at request + # time; the executor records it per output. Show the distinct resolved names. + resolved = sorted({e["model"] for e in out.values() if e.get("model")}) + resolved_note = f"; resolved model: `{'`, `'.join(resolved)}`" if resolved else "" + lines = [ MARKER, f"## Humanize skill benchmark — {'PASS ✅' if passed else 'FAIL ❌'}", "", f"{len(ids)} fixed AI-flavored inputs humanized with the PR's `humanize/SKILL.md` " - f"(executor: {args.executor_label}) and scored against the raw inputs as baseline.", + f"(executor: {args.executor_label}{resolved_note}) and scored against the raw " + "inputs as baseline.", "", - "**Verification model:** scores come from the official " - "[Binoculars](https://github.com/ahans30/Binoculars) zero-shot AI-text detector " + "**Scoring model:** the official " + "[Binoculars](https://github.com/ahans30/Binoculars) zero-shot scorer " "(Hans et al., ICML 2024) running the `TinyLlama-1.1B` base + `TinyLlama-1.1B-Chat` " "model pair.", "",