#+title: Work for robots
#+PROPERTY: header-args:sqlite :db /scratch/titan/apache2/cgit-logs.sqlite :colnames yes
#+PROPERTY: header-args :exports both :cache yes :eval no-export
#+HTML_HEAD: <script src="http://127.0.0.1:8095/skewer"></script>
#+HTML_HEAD: <link rel="stylesheet" href="static/uPlot.min.css" />
#+HTML_HEAD_EXTRA: <script src="static/uPlot.iife.min.js"></script>
#+HTML_HEAD_EXTRA: <script src="plots.js"></script>

I self-host my website to keep sovereignty and independence from large Internet
corporations. I don't have at the moment any analytics tool on this site.
However, some of my git repositories are also self-hosted, and there I have
access to the =Apache= logs. With all the =AI-hype= and coding assistants, I
wanted to have a look at how of my =software= are those AI companies taking.

Analyzing my log files in the period from =2024-12-31 23:00:35= till =2025-04-20
00:38:24= I'm surprised to see how dumb those AI companies are in the use of
their web crawlers.

Table [[top-users]] shows the top /users/ of my git repository. The leading AI
companies =OpenAI= and =Anthropic= with their respective bots =GPTBot= and
=ClaudeBot= simply dominate. I found it unbelievable that they could extract
about =≈7GiB= of data. That is a lot of Bandwidth out of my server for a few git
repositories and in a lightweight web interphase.

#+begin_src sqlite :exports results
--SELECT
    --count(*) AS Requests,
    --round(total (length) / 1024 / 1024, 1) "Tx MiB",
    ----0,
    --'Everybody else' AS "User Agent"
--FROM
    --logs
--WHERE
    --agentid NOT IN (143, 1, 19, 6, 4602, 3, 2, 4, 10306, 9)
    --AND path NOT LIKE '/ingrid/%'
--UNION
SELECT
    count(*) AS Requests,
    round(total (length) / 1024 / 1024, 1) "Tx MiB",
    --agentid,
    user_agent AS "User Agent"
FROM
    logs
    JOIN agent ON agent.id = logs.agentid
WHERE
    path NOT LIKE '/ingrid/%'
GROUP BY
    agentid
ORDER BY
    2 DESC
LIMIT 10
#+end_src

#+name: top-users
#+caption: Top 10 /users/ as self-identified by their /User Agent/ to the server
#+RESULTS[36d7b647efa39c3af86581279748a2bb53d034f3]:
| Requests | Tx MiB | User Agent                                                                                                                                                            |
|----------+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|  3572480 | 8819.6 | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                |
|  1617262 | 6766.3 | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                               |
|   273968 |  721.4 | Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)                                                                                                |
|    80159 |  498.3 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36                                                 |
|   207771 |  475.8 | Scrapy/2.11.2 (+https://scrapy.org)                                                                                                                                   |
|    69697 |  466.1 | Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot) |
|    59832 |  416.4 | Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)                                                                                                    |
|    14142 |   83.3 | Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)                  |
|     2500 |   53.7 | Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)                                                                                                      |
|     3578 |   30.9 | Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.7049.52 Mobile Safari/537.36 (compatible; GoogleOther)  |

#+begin_src sqlite :results value file :file top_agent_traffic.csv :exports none
SELECT
    date / 14400 * 14400 AS time, -- 4h bin
    count(*) AS requests,
    total (length) FILTER (WHERE agentid = 143) AS "OpenAI-GPTBot",
    total (length) FILTER (WHERE agentid = 1) AS "ClaudeBot",
    total (length) FILTER (WHERE agentid = 19) AS "Barkrowler",
    total (length) FILTER (WHERE agentid = 6) AS "Macintosh",
    total (length) FILTER (WHERE agentid = 4602) AS "Scrapy",
    total (length) FILTER (WHERE agentid = 3) AS "PetalBot",
    total (length) FILTER (WHERE agentid = 2) AS "AhrefsBot",
    total (length) FILTER (WHERE agentid = 4) AS "Bytespider",
    total (length) FILTER (WHERE agentid = 10306) AS "SeekportBot",
    total (length) FILTER (WHERE agentid = 9) AS "Google",
    total (length) FILTER (WHERE agentid not in (143, 1, 19, 6, 4602, 3, 2, 4, 10306, 9)) AS "Rest"
FROM
    logs
WHERE
    path NOT LIKE '/ingrid/%'
    AND date NOT NULL
GROUP BY
    time
#+end_src

#+RESULTS[87f78b2de4d43785f81e682925c6b6542d794883]:
[[file:top_agent_traffic.csv]]

This is reviewed in figure [[fig:agent-traffic]]

#+attr_html: :id agent-traffic
#+CAPTION: Example of the process
#+NAME: fig:agent-traffic
[[./jsfill.png]]

#+caption: Caption shared by both figures
#+begin_quote
teusta sat
#+end_quote

following theorem [[th:theproc]]

#+CAPTION: Example of the process
#+NAME: th:theproc
#+attr_html: :style display:flex; :id fun
#+begin_theorem
If an integer $n$ is greater than 2, then the equation $a^n + b^n = c^n$
has no solutions in non-zero integers $a$, $b$, and $c$.
#+end_theorem

look fig [[figblo]]

#+NAME: figblo
#+attr_html: :class tusi figblo
#+CAPTION: fun block
#+begin_figure
satore
#+end_figure


#+begin_src sqlite
SELECT
    round(total (length) / 1024, 2) "tx KiB",
    count(*) hits,
    agentid,
    user_agent
FROM
    logs
    JOIN agent ON agentid = id
    --where agentid in (
    --select id from agent
WHERE
    user_agent LIKE '%google%'
    --)
GROUP BY
    id
ORDER BY
    1 DESC
#+end_src


#+begin_src sqlite
SELECT
    --agent.id,
    user_agent as "User Agent",
    count(*) AS hits,
    round(total (length) / 1024, 1) AS "tx KiB"
FROM
    logs
    JOIN agent ON logs.agentid = agent.id
WHERE
    user_agent LIKE '%git%'
    and id != 10897
GROUP BY
    user_agent
order by total(length) desc
#+end_src

#+RESULTS[333fcbc738819c497f14b4445a3b45f391f0db7e]:
| User Agent                                                                       | hits |  tx KiB |
|----------------------------------------------------------------------------------+------+---------|
| git/2.40.3                                                                       | 1075 | 12821.3 |
| git/2.34.1                                                                       | 1149 |  3687.1 |
| Software Heritage dumb Git loader                                                |  337 |  2533.6 |
| git/2.48.1                                                                       |  115 |  1908.6 |
| Software Heritage cgit lister v6.9.3 (+https://www.softwareheritage.org/contact) |    8 |    21.3 |
| Software Heritage cgit lister v6.9.2 (+https://www.softwareheritage.org/contact) |    8 |    21.0 |
| git/dulwich/0.22.7                                                               |    2 |     8.5 |
| git/dulwich/0.22.6                                                               |    1 |     4.2 |
|----------------------------------------------------------------------------------+------+---------|
| Total                                                                            | 2695 | 21005.6 |




#+begin_src sqlite
SELECT
    status,
    count(*) as hits,
    round(total (length) / 1024 / 1024, 2) as "tx MB"
    -- round(total (total (length)) OVER (PARTITION BY status / 100)  / 1024 / 1024, 2) AS "total tx MB"
FROM
    logs
GROUP BY
    "status"
order by "tx MB" desc
#+end_src

#+RESULTS[6e8a6986b881fbf55f5e1328d505eb280c73f3b0]:
| status |    hits |    tx MB |
|--------+---------+----------|
|    200 | 4255831 | 13463.72 |
|    404 |  422650 |  1617.18 |
|    400 |    3087 |     3.89 |
|    500 |     790 |     1.37 |
|    302 |     202 |      0.5 |
|    304 |     240 |      0.3 |
|    206 |     297 |     0.26 |
|    403 |      29 |     0.11 |
|    301 |      23 |     0.08 |
|        |      90 |     0.06 |
|    421 |       7 |     0.03 |
|    408 |       3 |     0.01 |

#+begin_src sqlite
SELECT
    count(*),
    count(distinct ipid),
    count(distinct agentid),
    total (length),
    request_method,
    -- status,
    substr(path, 0, 50)
FROM
    logs
WHERE
    path NOT LIKE '/ingrid/%' AND
    status = 200
GROUP BY
    path
ORDER BY
    1 DESC
LIMIT 15
#+end_src

#+RESULTS[8469c661009196d66d76364873866923b89c8205]:
| count(*) | count(distinct ipid) | count(distinct agentid) | total (length) | request_method | substr(path, 0, 50)             |
|----------+----------------------+-------------------------+----------------+----------------+---------------------------------|
|     3540 |                 1779 |                     425 |     22903047.0 | GET            | /                               |
|     2171 |                 1563 |                      48 |      9744532.0 | GET            | /robots.txt                     |
|      855 |                  518 |                      50 |      4234481.0 | GET            | /favicon.ico                    |
|      334 |                  262 |                      31 |      1304042.0 | GET            | /cgit.css                       |
|      150 |                  116 |                      59 |      1030405.0 | GET            | /hugo-minimalist-theme/about/   |
|      144 |                    1 |                       1 |       854165.0 | GET            | /?XDEBUG_SESSION_START=phpstorm |
|      119 |                   69 |                      52 |       421470.0 | GET            | /cgit.png                       |
|       98 |                   36 |                      14 |       330051.0 | GET            | /hugo-minimalist-theme/         |
|       46 |                   31 |                      15 |       165446.0 | GET            | /homepage/                      |
|       42 |                   32 |                      13 |       157112.0 | GET            | /dotfiles/                      |
|       39 |                   34 |                      14 |       255204.0 | GET            | /hugo-minimalist-theme/log/     |
|       37 |                   30 |                      13 |       132913.0 | GET            | /hugo-minimalist-theme/tree/    |
|       37 |                   33 |                      11 |       103541.0 | GET            | /?s=name                        |
|       36 |                   33 |                      12 |       242095.0 | GET            | /dotfiles/log/                  |
|       35 |                   32 |                      14 |       124829.0 | GET            | /dotfiles/tree/                 |

#+begin_src sqlite
SELECT
    count(*) as hits,
    sum(length),
    agentid,
    path
    --substr(path, 0, 50)
FROM
    logs
WHERE
    path NOT LIKE '/ingrid/%'
    AND status = 404
GROUP BY
    path
ORDER BY
       hits desc
limit 10
#+end_src

#+RESULTS[3dbf9422fb49ecb2cae63c67615825ef1b1c30a9]:
| hits | sum(length) | agentid | path                                                                                |                    |
|------+-------------+---------+-------------------------------------------------------------------------------------+--------------------|
| 2815 |     8768239 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.RelPermalink%20%7D%7D       |                    |
| 1643 |     5653820 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                |                    |
| 1564 |     5049557 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.%20%7D%7D                   |                    |
| 1218 |     4488059 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$href%20%7D%7D               |                    |
| 1077 |     3930371 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.Permalink%20%7D%7D          |                    |
|  924 |     3522501 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Next.URL%20%7D%7D       |                    |
|  919 |     3500841 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Prev.URL%20%7D%7D       |                    |
|  818 |     3088624 |     143 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20if%20ne%20.MediaType.SubType |                    |
|  807 |     2530428 |     143 | /hugo-minimalist-theme/plain/layouts/%7B%7B%20.Type%20%7D%7D                        |                    |
|  750 |     3000020 |     143 | /hugo-minimalist-theme/plain/layouts/taxonomy/%7B%7B%20.Name%20                     | %20urlize%20%7D%7D |

#+begin_src sqlite :results value file :file fails.csv
SELECT
    count(*) as hits,
    sum(length),
    path
    --substr(path, 0, 50)
FROM
    logs
WHERE
    path NOT LIKE '/ingrid/%'
    AND status = 404
GROUP BY
    path
ORDER BY
       hits desc, path
limit 550
#+end_src

#+RESULTS[28ba3f6de596e93fd80baf9516e5ae32a8239c72]:
[[file:fails.csv]]

#+begin_src sqlite
select count(*), sum(length), path,user_agent from logs
join agent on agentid=id
WHERE
    path NOT LIKE '/ingrid/%'
    AND status = 404
group by path, agentid
ORDER BY
       sum(count(*)) over (partition by path) desc, 1 desc, path, agentid
limit 50
#+end_src

#+RESULTS[cb18bd88215d0873626e54f3a56ea4b6851c4138]:
| count(*) | sum(length) | path                                                                                | user_agent                                                                                                                                                                  |                                                                                                         |
|----------+-------------+-------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------|
|     2805 |     8732358 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.RelPermalink%20%7D%7D       | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        7 |       18923 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.RelPermalink%20%7D%7D       | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|        3 |       16958 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.RelPermalink%20%7D%7D       | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|     1629 |     5597913 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        7 |       25024 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|        3 |        8467 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com)                                                                                                                      |                                                                                                         |
|        1 |        2655 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|        1 |        6609 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | Mozilla/5.0 (Macintosh; PPC Mac OS X 10_7_9 rv:3.0; mt-MT) AppleWebKit/532.12.1 (KHTML, like Gecko) Version/5.0.3 Safari/532.12.1                                           |                                                                                                         |
|        1 |        6609 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | Mozilla/5.0 (X11; Linux i686; rv:1.9.6.20) Gecko/8855-02-04 23:24:49.455231 Firefox/3.6.10                                                                                  |                                                                                                         |
|        1 |        6543 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.URL%20%7D%7D                | Opera/9.30.(X11; Linux x86_64; tn-ZA) Presto/2.9.175 Version/11.00                                                                                                          |                                                                                                         |
|     1559 |     5018667 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.%20%7D%7D                   | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        4 |       23869 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.%20%7D%7D                   | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|        1 |        7021 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.%20%7D%7D                   | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|     1209 |     4437078 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$href%20%7D%7D               | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        5 |       27298 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$href%20%7D%7D               | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|        4 |       23683 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$href%20%7D%7D               | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|     1060 |     3867072 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.Permalink%20%7D%7D          | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|       12 |       41170 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.Permalink%20%7D%7D          | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|        3 |        8087 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.Permalink%20%7D%7D          | AliyunSecBot/Aliyun (AliyunSecBot@service.alibaba.com)                                                                                                                      |                                                                                                         |
|        2 |       14042 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.Permalink%20%7D%7D          | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|      916 |     3490491 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Next.URL%20%7D%7D       | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        7 |       29355 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Next.URL%20%7D%7D       | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|        1 |        2655 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Next.URL%20%7D%7D       | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|      912 |     3471446 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Prev.URL%20%7D%7D       | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        7 |       29395 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20$pag.Prev.URL%20%7D%7D       | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|      817 |     3081643 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20if%20ne%20.MediaType.SubType | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        1 |        6981 | /hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20if%20ne%20.MediaType.SubType | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|      798 |     2487903 | /hugo-minimalist-theme/plain/layouts/%7B%7B%20.Type%20%7D%7D                        | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)                                                                      |                                                                                                         |
|        5 |       26298 | /hugo-minimalist-theme/plain/layouts/%7B%7B%20.Type%20%7D%7D                        | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)                                                                     |                                                                                                         |
|        4 |       16227 | /hugo-minimalist-theme/plain/layouts/%7B%7B%20.Type%20%7D%7D                        | Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)                                                                                                                      |                                                                                                         |
|      745 |     2977378 | /hugo-minimalist-theme/plain/layouts/taxonomy/%7B%7B%20.Name%20                     | %20urlize%20%7D%7D                                                                                                                                                          | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)  |
|        5 |       22642 | /hugo-minimalist-theme/plain/layouts/taxonomy/%7B%7B%20.Name%20                     | %20urlize%20%7D%7D                                                                                                                                                          | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com) |
|       65 |      348406 | /.env                                                                               | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0                                                                                          |                                                                                                         |
|       63 |      310359 | /.env                                                                               | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36                                                                   |                                                                                                         |
|       58 |      310754 | /.env                                                                               | Mozilla/5.0 (Linux; U; Android 4.4.2; en-US; HM NOTE 1W Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/11.0.5.850 U3/0.8.0 Mobile Safari/534.30 |                                                                                                         |
|       52 |      301380 | /.env                                                                               | l9explore/1.2.2                                                                                                                                                             |                                                                                                         |
|       22 |      111334 | /.env                                                                               | Go-http-client/1.1                                                                                                                                                          |                                                                                                         |
|       18 |       38352 | /.env                                                                               | -                                                                                                                                                                           |                                                                                                         |
|       16 |       85808 | /.env                                                                               | python-requests/2.32.3                                                                                                                                                      |                                                                                                         |
|       16 |       36940 | /.env                                                                               | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36                                                         |                                                                                                         |
|        9 |       52245 | /.env                                                                               | Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0 abuse.xmco.fr                                                                          |                                                                                                         |
|        9 |       48190 | /.env                                                                               | Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50                                                   |                                                                                                         |
|        8 |       42840 | /.env                                                                               | python-requests/2.26.0                                                                                                                                                      |                                                                                                         |
|        6 |       32086 | /.env                                                                               | python-requests/2.31.0                                                                                                                                                      |                                                                                                         |
|        5 |       26773 | /.env                                                                               | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3                                                          |                                                                                                         |
|        3 |        8719 | /.env                                                                               | Mozilla/5.0 (iPhone; CPU iPhone OS 17_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Mobile/15E148 Safari/604.1                                 |                                                                                                         |
|        3 |       16043 | /.env                                                                               | python-httpx/0.28.1                                                                                                                                                         |                                                                                                         |
|        3 |       17375 | /.env                                                                               | Mozilla/5.0 (Linux; Android 9; Redmi Note 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.89 Mobile Safari/537.36                                                |                                                                                                         |
|        3 |       17375 | /.env                                                                               | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0                                             |                                                                                                         |
|        3 |       16139 | /.env                                                                               | python-requests/2.27.1                                                                                                                                                      |                                                                                                         |

#+begin_src sqlite :results value file :file leadfail.csv
SELECT
    datetime (date, 'unixepoch'),
    status,
    count(*) as hits,
    round(sum(length) / 1024, 2) as KB,
    user_agent
FROM
    logs
    JOIN agent ON agent.id = logs.agentid
WHERE
    path = '/hugo-minimalist-theme/plain/layouts/partials/%7B%7B%20.RelPermalink%20%7D%7D'
group by agentid
ORDER BY
    referrer
#+end_src

#+RESULTS[e3501152f1ec3e7ff822a6713736ff35fb0dcb46]:
[[file:leadfail.csv]]