unlob Docs
Console
Browse documentation

Go

net/http and encoding/json. No dependencies.

Types

package unlob

type Hit struct {
	ID       string `json:"id"`
	URL      string `json:"url"`
	Host     string `json:"host"`
	Title    string `json:"title"`
	Vertical string `json:"vertical"`
	// The passage text — usually the answer, not a preview of one.
	Snippet   string  `json:"snippet"`
	Score     float64 `json:"score"`
	HostRank  float64 `json:"host_rank"`
	Quality   int64   `json:"quality"`
	FetchedAt int64   `json:"fetched_at"`
	Source    string  `json:"source"`
	StoryID   string  `json:"story_id"`

	// Omitted when they do not apply. A missing PublishedAt means the page
	// declared no date — not that it was published at the epoch.
	GroupSize          *int     `json:"group_size,omitempty"`
	PublishedAt        *int64   `json:"published_at,omitempty"`
	IndependentSources *int64   `json:"independent_sources,omitempty"`
	Centrality         *float64 `json:"centrality,omitempty"`
	Topics             []string `json:"topics,omitempty"`
}

type SearchResponse struct {
	Vertical *string `json:"vertical"`
	Routed   bool    `json:"routed"`
	Mode     string  `json:"mode"`
	Total    int     `json:"total"`
	Results  []Hit   `json:"results"`
	// True when part of the corpus was unreachable. The answer is INCOMPLETE,
	// not merely short.
	Partial bool `json:"partial,omitempty"`
}

Pointers for the optional numerics on purpose: IndependentSources == nil means unknown, which is a different fact from zero.

The client

package unlob

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

var ErrQuotaExhausted = errors.New("unlob: monthly quota exhausted")

type Client struct {
	Key  string
	Base string
	HTTP *http.Client
}

func New(key string) *Client {
	return &Client{
		Key:  key,
		Base: "https://api.unlob.com",
		HTTP: &http.Client{Timeout: 20 * time.Second},
	}
}

func (c *Client) get(ctx context.Context, path string, q url.Values, out any) error {
	for attempt := 0; attempt < 4; attempt++ {
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.Base+path+"?"+q.Encode(), nil)
		if err != nil {
			return err
		}
		req.Header.Set("x-api-key", c.Key)

		resp, err := c.HTTP.Do(req)
		if err != nil {
			if attempt < 3 {
				sleep(ctx, attempt)
				continue
			}
			return err
		}

		if resp.StatusCode == http.StatusTooManyRequests {
			after := resp.Header.Get("retry-after")
			body, _ := io.ReadAll(resp.Body)
			resp.Body.Close()
			if after == "" {
				// No retry-after means a hard-capped monthly quota, not a rate
				// limit. Retrying is a loop that cannot succeed.
				return fmt.Errorf("%w: %s", ErrQuotaExhausted, body)
			}
			secs, _ := strconv.Atoi(after)
			select {
			case <-time.After(time.Duration(secs) * time.Second):
			case <-ctx.Done():
				return ctx.Err()
			}
			continue
		}

		if (resp.StatusCode >= 500 || resp.StatusCode == 408) && attempt < 3 {
			resp.Body.Close()
			sleep(ctx, attempt)
			continue
		}

		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			body, _ := io.ReadAll(resp.Body)
			return fmt.Errorf("unlob %d: %s", resp.StatusCode, body)
		}
		return json.NewDecoder(resp.Body).Decode(out)
	}
	return errors.New("unlob: exhausted retries")
}

// Full jitter. Without it, every client that failed together retries together
// and re-creates the spike that broke the thing they are waiting for.
func sleep(ctx context.Context, attempt int) {
	d := time.Duration(rand.Float64() * float64(int64(time.Second)<<attempt))
	select {
	case <-time.After(d):
	case <-ctx.Done():
	}
}

func (c *Client) Search(ctx context.Context, q url.Values) (*SearchResponse, error) {
	var out SearchResponse
	if err := c.get(ctx, "/search", q, &out); err != nil {
		return nil, err
	}
	return &out, nil
}

Using it

func main() {
	c := unlob.New(os.Getenv("UNLOB_API_KEY"))

	q := url.Values{}
	q.Set("q", "how does tokio schedule tasks")
	q.Set("vertical", "code")
	q.Set("collapse", "story")
	q.Set("limit", "5")

	body, err := c.Search(context.Background(), q)
	if errors.Is(err, unlob.ErrQuotaExhausted) {
		log.Fatal("out of quota for this period")
	} else if err != nil {
		log.Fatal(err)
	}

	if body.Partial {
		// A 200 that means the answer is incomplete.
		log.Println("WARNING: incomplete — part of the corpus was unreachable")
	}

	for _, h := range body.Results {
		fmt.Printf("%s — %s\n%s\n\n", h.Title, h.URL, h.Snippet)
	}
}

Filters

url.Values carries them all. IN-lists are comma-separated:

q.Set("tld", "gov,edu")
q.Set("min_independent_sources", "2")
q.Set("min_host_rank", "0.6")
q.Set("sort", "centrality")

Next