From dc45b56759e38da793f8976c94b6a798096c968e Mon Sep 17 00:00:00 2001 From: Oscar Najera Date: Wed, 29 Nov 2023 04:21:37 +0100 Subject: Semantic search server and client --- scratch/semgrep/semgrep.el | 116 +++++++++++++++++++++++++++++++++++++++++++++ scratch/semgrep/server.py | 92 +++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 scratch/semgrep/semgrep.el create mode 100644 scratch/semgrep/server.py diff --git a/scratch/semgrep/semgrep.el b/scratch/semgrep/semgrep.el new file mode 100644 index 0000000..5752a20 --- /dev/null +++ b/scratch/semgrep/semgrep.el @@ -0,0 +1,116 @@ +;;; semgrep.el --- Semantic search -*- lexical-binding: t; -*- +;; +;; Copyright (C) 2023 Óscar Nájera +;; +;; Author: Óscar Nájera +;; Maintainer: Óscar Nájera +;; Created: November 07, 2023 +;; Modified: November 07, 2023 +;; Version: 0.0.1 +;; Keywords: abbrev bib c calendar comm convenience data docs emulations extensions faces files frames games hardware help hypermedia i18n internal languages lisp local maint mail matching mouse multimedia news outlines processes terminals tex tools unix vc wp +;; Homepage: https://github.com/titan/semgrep +;; Package-Requires: ((emacs "27.1")) +;; +;; This file is not part of GNU Emacs. +;; +;;; Commentary: +;; +;; semantically search on my database by paragraph +;; +;;; Code: +(require 'url) +(require 'org-element) +(require 'org-roam-db) +(require 'dash) + +(defun semgrep--connect (method data) + (let ((url-request-method "POST") + (url-request-extra-headers '(("Content-Type" . "application/json"))) + (url-request-data (encode-coding-string + (json-serialize `(,method ,data)) + 'utf-8))) + (with-current-buffer + (url-retrieve-synchronously "http://localhost:8080") + (goto-char url-http-end-of-headers) + (json-read)))) + +(defun semgrep--get-node-id (paragraph &optional default) + (thread-first + (org-element-map + (org-element-property :parent paragraph) + 'node-property + (lambda (np) + (org-element-property :value np))) + (car) + (org-string-nw-p) + (or default))) + +(defun semgrep--prepare-paragraph (file-id) + (lambda (paragraph) + (list + :document (substring-no-properties (org-element-interpret-data paragraph)) + :metadata (list :start-point + (org-element-property :begin paragraph) + :node-id + (semgrep--get-node-id paragraph file-id))))) + +(defun semgrep--add-buffer () + (interactive) + (if (eq major-mode 'org-mode) + (-some--> + (org-element-map + (org-element-parse-buffer) + 'paragraph + (semgrep--prepare-paragraph (org-id-get (point-min)))) + (cl-coerce it 'vector) + ;; (json-serialize it) + ;; (f-write it 'utf-8 "/tmp/out.json") + ;; (message "%S" it) + (semgrep--connect :store it)) + (user-error "This only works on org-mode"))) + +(defun semgrep--roam-data (entries) + (thread-last + (cl-mapcar (lambda (meta) + (alist-get 'node-id meta)) + entries) + (delete-dups) + (vconcat) + (org-roam-db-query [:select [id title] + :from nodes + :where (in id $v1)]))) + +(defun semgrep-search (text) + (interactive (list (or (thing-at-point 'paragraph) + (read-from-minibuffer "What are you looking for? ")))) + (-let (((&alist 'distances 'documents 'metadatas) + (semgrep--connect :query text))) + (with-current-buffer (get-buffer-create "*Semantic Search*") + (erase-buffer) + (org-mode) + (insert "#+title: Looking for:\n" text "\n") + (cl-mapc + (lambda (entry-distances entry-document entry-metadatas) + (let ((data (semgrep--roam-data entry-metadatas))) + (cl-mapc + (lambda (d paragraph meta) + (let* ((node-id (or (alist-get 'node-id meta) "")) + (title (cadr (assoc node-id data #'string=)))) + (unless (zerop d) + (insert + (format "* [[id:%s][%s]]\n" node-id title) + "- Distance :: " (number-to-string d) "\n" + "- point :: " (number-to-string (or (alist-get 'start-point meta) -1)) "\n" + (string-trim paragraph) ?\n)))) + entry-distances entry-document entry-metadatas))) + distances documents metadatas) + (goto-char (point-min)) + (display-buffer (current-buffer))))) + +;; (org-roam-dolist-with-progress (file (org-roam-list-files)) +;; "importing to semantic search" +;; (org-roam-with-file file nil +;; (semgrep--add-buffer))) + +(provide 'semgrep) +;;; semgrep.el ends here diff --git a/scratch/semgrep/server.py b/scratch/semgrep/server.py new file mode 100644 index 0000000..6a8648b --- /dev/null +++ b/scratch/semgrep/server.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +from http.server import BaseHTTPRequestHandler, HTTPServer +import chromadb +import collections +import hashlib +import json + + +def checksum(string): + sha256 = hashlib.sha256() + sha256.update(string.encode("utf-8")) + return sha256.hexdigest()[:32] + + +def ensure_list(data): + if isinstance(data, str): + return [data] + if isinstance(data, list): + if all(isinstance(l, str) for l in data): + return data + raise ValueError("Data must be a list of strings") + + +class MyRequestHandler(BaseHTTPRequestHandler): + def do_POST(self): + content_length = int(self.headers["Content-Length"]) + post_data = self.rfile.read(content_length).decode("utf-8") + + try: + data = json.loads(post_data) + # Process the JSON data + response_message = f"Received POST request with data: '{data}'\n" + except ValueError: + response_message = "Invalid JSON data" + self.send_response(400) + + if query := data.get("query"): + response = collection.query(query_texts=ensure_list(query)) + elif paragraph := data.get("store"): + data, metadata = drop_duplicates(paragraph) + collection.add( + documents=data, metadatas=metadata, ids=[checksum(l) for l in data] + ) + response = {"added": data} + else: + raise ValueError(f"Used wrong method. Sent: {data.keys()}") + + response_message = json.dumps(response) + + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + self.wfile.write(response_message.encode("utf-8")) + + +def run_server(port=8080): + server_address = ("", port) + httpd = HTTPServer(server_address, MyRequestHandler) + print(f"Server running on port {port}") + httpd.serve_forever() + + +def drop_duplicates(paragraph): + data = [data["document"] for data in paragraph] + metadata = [data["metadata"] for data in paragraph] + dups = (x for x, count in collections.Counter(data).items() if count > 1) + to_drop = [] + for no in dups: + to_drop.extend([i for i, d in enumerate(data) if d == no][1:]) + to_drop.sort(reverse=True) + for index in to_drop: + data.pop(index) + metadata.pop(index) + return data, metadata + + +def test(): + sample = [ + {"document": "Hello", "metadata": 5}, + {"document": "World", "metadata": 8}, + {"document": "Hello", "metadata": 6}, + {"document": "Good", "metadata": 3}, + {"document": "World", "metadata": 9}, + ] + + assert drop_duplicates(sample) == (["Hello", "World", "Good"], [5, 8, 3]) + + +if __name__ == "__main__": + client = chromadb.PersistentClient(path="./semgrep") + collection = client.get_or_create_collection("org-roam") + run_server() -- cgit v1.2.3