diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rwxr-xr-x | bin/backup.sh | 9 | ||||
| -rw-r--r-- | config/calendars.conf (renamed from calendars.conf) | 0 | ||||
| -rw-r--r-- | config/doom/config.org | 85 | ||||
| -rw-r--r-- | config/doom/init.el | 2 | ||||
| -rw-r--r-- | config/doom/snippets/org-mode/daily | 25 | ||||
| -rw-r--r-- | config/pikaur.conf | 3 | ||||
| -rw-r--r-- | config/shepherd/init.d/cardano.scm | 10 | ||||
| -rw-r--r-- | config/shepherd/init.d/my-daemons.scm | 14 | ||||
| -rw-r--r-- | config/shepherd/init.scm | 2 | ||||
| -rw-r--r-- | home-dots/dot-bashrc | 10 | ||||
| -rw-r--r-- | home-dots/dot-gitconfig | 6 | ||||
| -rwxr-xr-x | install.scm | 6 | ||||
| -rw-r--r-- | lib/guile/ffi/notmuch.scm | 62 | ||||
| -rw-r--r-- | scratch/semgrep/requirements.txt | 2 | ||||
| -rw-r--r-- | scratch/semgrep/semantic-search.el | 148 | ||||
| -rw-r--r-- | scratch/semgrep/server.py | 116 | ||||
| -rw-r--r-- | scratch/semgrep/tests.py | 15 | ||||
| -rw-r--r-- | scratch/semgrep/utils.py | 21 |
19 files changed, 457 insertions, 80 deletions
@@ -1 +1,2 @@ /elisp/*.elc +__pycache__ diff --git a/bin/backup.sh b/bin/backup.sh index 8887095..9f53249 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -29,8 +29,13 @@ backup_local() { --exclude '*/mpd/log' \ --exclude '*/elpa' \ --exclude '*.elc' \ + --exclude '*/*.o' \ + --exclude '*/guix/*/*.go' \ + --exclude '*/guix/*/*.Po' \ + --exclude '*/guix/*/*.m4' \ --exclude '*/.local/straight' \ --exclude '*/phd/simulation_data' \ + --exclude '*/semantic-roam' \ "$REPOSITORY::{hostname}-{user}-{utcnow:%Y-%m-%dT%H:%M:%S}" \ "$@" # all the dirs to bac @@ -50,6 +55,8 @@ backup_local Admin/sarah/borg/ingrid ssh://borgbackup@sarah/./repos/ingrid "${BA DISK=/run/media/titan/ext_backup if [ -d "$DISK" ]; then LARGER_PATHS=("$DISK/personal/Pictures/" "$DISK/personal/Libros/" "$DISK/personal/dev/" "$DISK/Archives/") - backup_local borgbackup ssh://backup/media/Backup/daily_backup/ "${BASIC_PATHS[@]}" "${LARGER_PATHS[@]}" backup_local Admin/sarah/borg/oscar ssh://borgbackup@sarah/./repos/oscar "${LARGER_PATHS[@]}" + if ping -c 1 -W 1 192.168.178.128 >/dev/null; then + backup_local borgbackup ssh://backup/media/Backup/daily_backup/ "${BASIC_PATHS[@]}" "${LARGER_PATHS[@]}" + fi fi diff --git a/calendars.conf b/config/calendars.conf index f09945f..f09945f 100644 --- a/calendars.conf +++ b/config/calendars.conf diff --git a/config/doom/config.org b/config/doom/config.org index c387f0e..577f1f9 100644 --- a/config/doom/config.org +++ b/config/doom/config.org @@ -11,6 +11,22 @@ #+begin_src emacs-lisp (setq doom-font (font-spec :family "DejaVu Sans Mono" :size 18 :weight 'semi-light)) (setq doom-theme 'doom-gruvbox) +(add-hook 'prog-mode-hook 'prettify-symbols-mode) + +(defun on/prettify-symbols () + "Common added symbols." + (mapc (lambda (p) + (cl-pushnew p prettify-symbols-alist :test #'equal)) + '(("<=" . ?≤) + (">=" . ?≥) + ("->" . ?→) + ("->>" . ?↠) + ("=>" . ?⇒) + ("/=" . ?≠) ("!=" . ?≠) + ("::" . ?∷) + ("lambda" . ?λ)))) + +(add-hook 'prog-mode-hook 'on/prettify-symbols) #+end_src This determines the style of line numbers in effect. If set to `nil', line numbers are disabled. For relative line numbers, set this to `relative'. @@ -88,6 +104,9 @@ Because I always need help and it should come up quickly (emacs-lisp-mode) (display-buffer (current-buffer)))) #+end_src +#+begin_src emacs-lisp + (add-hook 'after-save-hook 'executable-make-buffer-file-executable-if-script-p) +#+end_src * Dictionary #+begin_src emacs-lisp :tangle "packages.el" (package! lexic) @@ -120,25 +139,19 @@ Shamelessly copied from https://tecosaur.github.io/emacs-config/#dictionary `("e" "Event" entry (file ,(expand-file-name "caldav.org" org-directory)) "* %?\n%^T\n%i\n%a")) - (add-to-list 'org-capture-templates - `("m" "Meetup" entry (file ,(expand-file-name - "caldav.org" org-directory)) - "* %?%:description \n%i\n%l")) - (add-to-list 'org-capture-templates - `("i" "Isar-Speak Contact" - entry (file ,(expand-file-name "Isar-Speak.org" org-directory)) - "* INTRS [[notmuch-search:%:fromaddress][%:fromname]] -:PROPERTIES: -:EMAIL: %:fromaddress -:END: -Invited: %^t -")) (setq org-roam-capture-templates '(("d" "default" plain "%i\n%?" :target (file+head "%<%Y%m%d%H%M%S>-${slug}.org" "#+title: ${title}") :unnarrowed t)) org-roam-capture-ref-templates org-roam-capture-templates)) + +(use-package! semantic-search + :after org + :load-path "~/dev/dotfiles/scratch/semgrep" + :commands (semantic-search) + :config + (setq semantic-server-url "http://localhost:8080")) #+end_src ** Alerts This is to set the reminders of calendar events. @@ -230,28 +243,37 @@ this new window. #+begin_src emacs-lisp (after! elfeed (setq elfeed-feeds - '("https://sachachua.com/blog/feed" + '("http://feeds.feedburner.com/MeltingAsphalt" "http://feeds.feedburner.com/TroyHunt" - "https://serokell.io/blog.rss.xml" - "https://www.tweag.io/rss.xml" "http://kitchingroup.cheme.cmu.edu/blog/feed/index.xml" - "http://planet.emacsen.org/atom.xml" "http://www.howardism.org/index.xml" - "https://reyify.com/api/rss" - "https://guix.gnu.org/feeds/blog.atom" "https://ag91.github.io/rss.xml" + "https://architectelevator.com/feed.xml" + "https://bitcoinops.org/feed.xml" + "https://cyberwardog.blogspot.com/feeds/posts/default" "https://daryl.wakatara.com/rss.xml" + "https://guix.gnu.org/feeds/blog.atom" + "https://hboeck.de/rss.xml" + "https://jackrusher.com/feed.xml" "https://nullprogram.com/feed/" + "https://oscarnajera.com/post/index.xml" + "https://paulstamatiou.com/posts.xml" + "https://planet.archlinux.org/atom.xml" + "https://reyify.com/api/rss" + "https://sachachua.com/blog/feed" + "https://serokell.io/blog.rss.xml" + "https://stevelosh.com/rss.xml" + "https://www.archlinux.org/feeds/news/" + "https://www.n16f.net/blog/index.xml" "https://www.schneier.com/feed/atom" - "https://stevelosh.com/rss.xml"))) - + "https://www.tweag.io/rss.xml"))) #+end_src * Email ** Notmuch #+begin_src emacs-lisp :tangle "packages.el" -(package! notmuch :pin "7b5921877e748338359a25dae578771f768183af") -(package! org-mime :pin "a7bf07316f93015e4f853ea0fc5b8d05b4a7695d") -(package! ol-notmuch :pin "126fb446d8fa9e54cf21103afaf506fd81273c02") +(package! notmuch :pin "c769658360e10a6d01a4134e680e2f498741bc5c") +(package! org-mime :pin "cc00afcf0291633324364c1c83bfe2833cfdc1bf") +(package! ol-notmuch :pin "781c3518a537da2a8b5e8a4424f9441df463a147") #+end_src #+BEGIN_SRC emacs-lisp (use-package! notmuch @@ -604,6 +626,18 @@ Emacs mode for managing ledger text files '((sbcl-large ("sbcl" "--dynamic-space-size" "2048")) (sbcl ("sbcl"))))) #+end_src +** C +#+begin_src emacs-lisp +(after! lsp-clangd + (setq lsp-clients-clangd-args + '("-j=3" + "--background-index" + "--clang-tidy" + "--completion-style=detailed" + "--header-insertion=never" + "--header-insertion-decorators=0")) + (set-lsp-priority! 'clangd 2)) +#+end_src * Video Editing #+begin_src emacs-lisp :tangle "packages.el" @@ -626,12 +660,11 @@ Emacs mode for managing ledger text files (package! json-rpc) #+end_src #+begin_src emacs-lisp -(use-package! json-rpc) - (use-package! btc-explorer :load-path "~/dev/emacs-lisp/btc-explorer/" :commands (on/bitcoin-rpc-connect) :config + (use-package json-rpc) (defun on/bitcoin-rpc-connect () (interactive) (let* ((networks '(:mainnet 8332 diff --git a/config/doom/init.el b/config/doom/init.el index b34e18b..cd9d65d 100644 --- a/config/doom/init.el +++ b/config/doom/init.el @@ -157,7 +157,7 @@ ;;php ; perl's insecure younger brother ;;plantuml ; diagrams for confusing people more ;;purescript ; javascript, but functional - (python +lsp +ipython) ; beautiful is better than ugly + (python +lsp +ipython +pyright +tree-sitter) ; beautiful is better than ugly ;;qt ; the 'cutest' gui framework ever ;;racket ; a DSL for DSLs ;;raku ; the artist formerly known as perl6 diff --git a/config/doom/snippets/org-mode/daily b/config/doom/snippets/org-mode/daily index f0f24fd..57e60c2 100644 --- a/config/doom/snippets/org-mode/daily +++ b/config/doom/snippets/org-mode/daily @@ -2,17 +2,18 @@ # name: daily # key: daily # -- -- Amazing things that happened yesterday -- How could I have made yesterday even better? -- Daily affirmations -- What would make today great - -- What am I *happy* about in my life now? What about that makes me happy? How does that make me feel? -- What am I *excited* about in my life now? What about that makes me excited? How does that make me feel? -- What am I *proud* about in my life now? What about that makes me proud? How does that make me feel? -- What am I *grateful* for in my life now? what about that makes me grateful? how does that make me feel? -- What am I *enjoying most* in my life right now? what about that do I enjoy? How does that make me feel? -- What am I *committed to* in my life right now? What about that make me committed? How does that make me feel? -- Who do I love? Who loves me? What about that makes me loving? How does that make me feel? +* Routine Review +** Amazing things that happened yesterday +** How could I have made yesterday even better? +** Daily affirmations +** What would make today great +** What makes me feel? And what about that makes me feel that way? :feelings: +- happy :: about in my life now? What about that makes me happy? How does that make me feel? +- excited :: about in my life now? What about that makes me excited? How does that make me feel? +- proud :: about in my life now? What about that makes me proud? How does that make me feel? +- grateful :: for in my life now? what about that makes me grateful? how does that make me feel? +- enjoying most :: in my life right now? what about that do I enjoy? How does that make me feel? +- committed :: to in my life right now? What about that make me committed? How does that make me feel? +- love :: Who do I love? Who loves me? What about that makes me loving? How does that make me feel? - [ ] Timebox day
\ No newline at end of file diff --git a/config/pikaur.conf b/config/pikaur.conf index 1953fba..6c8e171 100644 --- a/config/pikaur.conf +++ b/config/pikaur.conf @@ -40,6 +40,9 @@ warnaboutpackageupdates = sudoloopinterval = 59 pacmanpath = pacman privilegeescalationtool = sudo +privilegeescalationtarget = pikaur +userid = 0 +preserveenv = PKGDEST,VISUAL,EDITOR,http_proxy,https_proxy,ftp_proxy,HTTP_PROXY,HTTPS_PROXY,FTP_PROXY,ALL_PROXY [network] aururl = https://aur.archlinux.org diff --git a/config/shepherd/init.d/cardano.scm b/config/shepherd/init.d/cardano.scm index bc89c84..1c5b731 100644 --- a/config/shepherd/init.d/cardano.scm +++ b/config/shepherd/init.d/cardano.scm @@ -30,7 +30,7 @@ #:provides '(preview-wallet) #:start (make-forkexec-constructor (list "cardano-wallet" - ;;(string-append (getenv "HOME") "/.cabal/bin/cardano-wallet") + ;;(string-append (getenv "HOME") "/.cabal/bin/cardano-wallet") "serve" "--port" "8091" "--node-socket" "/tmp/nina-pr.socket" @@ -48,10 +48,10 @@ #:start (make-forkexec-constructor (list ;;(string-append (getenv "HOME") "/.cabal/bin/cardano-wallet") "cardano-wallet" - "serve" - "--node-socket" "/run/cardano-node-mainnet/socket" - "--database" (string-append (getenv "HOME") "/test-cardano/main/wallet") - "--mainnet") + "serve" + "--node-socket" "/run/cardano-node-mainnet/socket" + "--database" (string-append (getenv "HOME") "/test-cardano/main/wallet") + "--mainnet") #:log-file (string-append (getenv "HOME") "/test-cardano/main/log/wallet.log")) #:stop (make-kill-destructor 2) ;; 2 is SIGINT - interupt process stream, ctrl-C #:requires '())) diff --git a/config/shepherd/init.d/my-daemons.scm b/config/shepherd/init.d/my-daemons.scm index 6388850..1b5daf8 100644 --- a/config/shepherd/init.d/my-daemons.scm +++ b/config/shepherd/init.d/my-daemons.scm @@ -16,3 +16,17 @@ #:log-file "/tmp/habits.log") #:stop (make-kill-destructor 2) ;; 2 is SIGINT - interupt process stream, ctrl-C #:respawn? #t)) + +(let ((workdir (string-append (getenv "HOME") "/dev/dotfiles/scratch/semgrep/"))) + (register-services + (make <service> + #:docstring "Roam semantic DB" + #:provides '(roam-semantic-db) + #:start (make-forkexec-constructor + (list "python" + (string-append workdir "server.py")) + #:directory workdir + #:environment-variables (list (string-append "PATH=" (getenv "HOME") "/.virtualenvs/language/bin")) + #:log-file "/tmp/roam-semantic.log") + #:stop (make-kill-destructor 2) ;; 2 is SIGINT - interupt process stream, ctrl-C + #:respawn? #t))) diff --git a/config/shepherd/init.scm b/config/shepherd/init.scm index c94b034..3d47bd4 100644 --- a/config/shepherd/init.scm +++ b/config/shepherd/init.scm @@ -22,4 +22,4 @@ ;; Services to start when shepherd starts: ;; Add the name of each service that should be started to the list ;; below passed to 'for-each'. -(for-each start '(habit-logs)) +(for-each start '(habit-logs roam-semantic-db)) diff --git a/home-dots/dot-bashrc b/home-dots/dot-bashrc index 098d855..5488946 100644 --- a/home-dots/dot-bashrc +++ b/home-dots/dot-bashrc @@ -14,11 +14,8 @@ HISTCONTROL=ignorespace # enable color support of ls and also add handy aliases if [ -x /usr/bin/dircolors ]; then - test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + # test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" alias ls='ls --color=auto' - #alias dir='dir --color=auto' - #alias vdir='vdir --color=auto' - alias grep='grep --color=auto' alias fgrep='fgrep --color=auto' alias egrep='egrep --color=auto' @@ -41,7 +38,7 @@ git_current_branch() { python_current_venv() { local orange="\[\e[0;91m\]" if [[ -n ${VIRTUAL_ENV} ]]; then - echo " ${orange} $(basename ${VIRTUAL_ENV})" + echo " ${orange} $(basename "${VIRTUAL_ENV}")" fi } @@ -73,7 +70,8 @@ export EDITOR='emacsclient' if [ -z "$SSH_CLIENT" ]; then unset SSH_AGENT_PID if [ "${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then - export SSH_AUTH_SOCK="$(gpgconf --list-dirs agent-ssh-socket)" + SSH_AUTH_SOCK="$(gpgconf --list-dirs agent-ssh-socket)" + export SSH_AUTH_SOCK fi fi diff --git a/home-dots/dot-gitconfig b/home-dots/dot-gitconfig index c080709..cacdd3e 100644 --- a/home-dots/dot-gitconfig +++ b/home-dots/dot-gitconfig @@ -23,5 +23,11 @@ [diff] algorithm = histogram +[sendemail] + smtpEncryption = tls + smtpServer = mail.oscarnajera.com + smtpUser = hi@oscarnajera.com + smtpServerPort = 587 + [includeIf "gitdir:~/bytegit/**"] path = ~/bytegit/.gitconfig diff --git a/install.scm b/install.scm index c9e60ec..acabf6e 100755 --- a/install.scm +++ b/install.scm @@ -4,6 +4,7 @@ (use-modules (ice-9 and-let-star) + (ice-9 format) (ice-9 ftw) (ice-9 match) (ice-9 popen) @@ -168,8 +169,8 @@ file-name)) (define (format-config-entry item) - (format #t "[~a]~%" (car item)) - (format #t "~:{~a=~@{~a~^;~}~%~}~%" (cdr item))) + (format #t "[~a]~%" (car item)) + (format #t "~:{~a=~@{~a~^;~}~%~}~%" (cdr item))) (define (mail-config) (log-msg 'INFO "Configuring mail") @@ -212,7 +213,6 @@ (define (main args) (setup-logging) (log-msg 'INFO "Symlink PIM") - (config-links "calendars.conf" "~/.calendars.conf" #f) (symlink-tree "home-dots" "~/") (config-links "gnupg/gpg.conf" "~/.gnupg/gpg.conf" #f) (config-links "gnupg/gpg-agent.conf" "~/.gnupg/gpg-agent.conf" #f) diff --git a/lib/guile/ffi/notmuch.scm b/lib/guile/ffi/notmuch.scm index 4c02592..5c6fb6e 100644 --- a/lib/guile/ffi/notmuch.scm +++ b/lib/guile/ffi/notmuch.scm @@ -76,7 +76,8 @@ (define-public (unwrap-notmuch_status_t n) (cond ((symbol? n) - (or (assq-ref notmuch_status_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_status_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_status_t v) @@ -280,7 +281,8 @@ (define-public (unwrap-notmuch_database_mode_t n) (cond ((symbol? n) - (or (assq-ref notmuch_database_mode_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_database_mode_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_database_mode_t v) @@ -854,7 +856,8 @@ (define-public (unwrap-notmuch_query_syntax_t n) (cond ((symbol? n) - (or (assq-ref notmuch_query_syntax_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_query_syntax_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_query_syntax_t v) @@ -902,7 +905,8 @@ (define-public (unwrap-notmuch_sort_t n) (cond ((symbol? n) - (or (assq-ref notmuch_sort_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_sort_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_sort_t v) @@ -954,7 +958,8 @@ (define-public (unwrap-notmuch_exclude_t n) (cond ((symbol? n) - (or (assq-ref notmuch_exclude_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_exclude_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_exclude_t v) @@ -1617,7 +1622,8 @@ (define-public (unwrap-notmuch_message_flag_t n) (cond ((symbol? n) - (or (assq-ref notmuch_message_flag_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_message_flag_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_message_flag_t v) @@ -2492,6 +2498,7 @@ ;; NOTMUCH_CONFIG_USER_NAME, ;; NOTMUCH_CONFIG_AUTOCOMMIT, ;; NOTMUCH_CONFIG_EXTRA_HEADERS, +;; NOTMUCH_CONFIG_INDEX_AS_TEXT, ;; NOTMUCH_CONFIG_LAST, ;; } notmuch_config_key_t; (define notmuch_config_key_t-enum-nvl @@ -2509,7 +2516,8 @@ (NOTMUCH_CONFIG_USER_NAME . 10) (NOTMUCH_CONFIG_AUTOCOMMIT . 11) (NOTMUCH_CONFIG_EXTRA_HEADERS . 12) - (NOTMUCH_CONFIG_LAST . 13)) + (NOTMUCH_CONFIG_INDEX_AS_TEXT . 13) + (NOTMUCH_CONFIG_LAST . 14)) ) (define notmuch_config_key_t-enum-vnl (map (lambda (pair) (cons (cdr pair) (car pair))) @@ -2517,7 +2525,8 @@ (define-public (unwrap-notmuch_config_key_t n) (cond ((symbol? n) - (or (assq-ref notmuch_config_key_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_config_key_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_config_key_t v) @@ -2826,7 +2835,8 @@ (define-public (unwrap-notmuch_decryption_policy_t n) (cond ((symbol? n) - (or (assq-ref notmuch_decryption_policy_t-enum-nvl n) (error "bad arg"))) + (or (assq-ref notmuch_decryption_policy_t-enum-nvl n) + (throw 'ffi-help-error "bad arg: ~A" n))) ((integer? n) n) (else (error "bad arg")))) (define-public (wrap-notmuch_decryption_policy_t v) @@ -2907,7 +2917,8 @@ (NOTMUCH_DECRYPT_AUTO . 2) (NOTMUCH_DECRYPT_TRUE . 1) (NOTMUCH_DECRYPT_FALSE . 0) - (NOTMUCH_CONFIG_LAST . 13) + (NOTMUCH_CONFIG_LAST . 14) + (NOTMUCH_CONFIG_INDEX_AS_TEXT . 13) (NOTMUCH_CONFIG_EXTRA_HEADERS . 12) (NOTMUCH_CONFIG_AUTOCOMMIT . 11) (NOTMUCH_CONFIG_USER_NAME . 10) @@ -2922,7 +2933,8 @@ (NOTMUCH_CONFIG_MAIL_ROOT . 1) (NOTMUCH_CONFIG_DATABASE_PATH . 0) (NOTMUCH_CONFIG_FIRST . 0) - (NOTMUCH_CONFIG_LAST . 13) + (NOTMUCH_CONFIG_LAST . 14) + (NOTMUCH_CONFIG_INDEX_AS_TEXT . 13) (NOTMUCH_CONFIG_EXTRA_HEADERS . 12) (NOTMUCH_CONFIG_AUTOCOMMIT . 11) (NOTMUCH_CONFIG_USER_NAME . 10) @@ -2975,7 +2987,9 @@ (NOTMUCH_STATUS_NO_DATABASE . 20) (NOTMUCH_STATUS_NO_CONFIG . 19) (NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL . 18) - (NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION . 17) + (NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION + . + 17) (NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL . 16) (NOTMUCH_STATUS_ILLEGAL_ARGUMENT . 15) (NOTMUCH_STATUS_IGNORED . 14) @@ -3040,19 +3054,17 @@ (else (error "type mismatch")))) (define ffi-notmuch-types - '((pointer . "notmuch_database_t") "notmuch_database_t" - (pointer . "notmuch_query_t") "notmuch_query_t" - (pointer . "notmuch_threads_t") "notmuch_threads_t" - (pointer . "notmuch_thread_t") "notmuch_thread_t" - (pointer . "notmuch_messages_t") "notmuch_messages_t" - (pointer . "notmuch_message_t") "notmuch_message_t" - (pointer . "notmuch_tags_t") "notmuch_tags_t" - (pointer . "notmuch_directory_t") "notmuch_directory_t" - (pointer . "notmuch_filenames_t") "notmuch_filenames_t" - (pointer . "notmuch_config_list_t") "notmuch_config_list_t" - (pointer . "notmuch_config_values_t") "notmuch_config_values_t" - (pointer . "notmuch_config_pairs_t") "notmuch_config_pairs_t" - (pointer . "notmuch_indexopts_t") "notmuch_indexopts_t" "notmuch_compact_status_cb_t" + '((pointer . "notmuch_database_t") "notmuch_database_t" (pointer . + "notmuch_query_t") "notmuch_query_t" (pointer . "notmuch_threads_t") + "notmuch_threads_t" (pointer . "notmuch_thread_t") "notmuch_thread_t" + (pointer . "notmuch_messages_t") "notmuch_messages_t" (pointer . + "notmuch_message_t") "notmuch_message_t" (pointer . "notmuch_tags_t") + "notmuch_tags_t" (pointer . "notmuch_directory_t") "notmuch_directory_t" + (pointer . "notmuch_filenames_t") "notmuch_filenames_t" (pointer . + "notmuch_config_list_t") "notmuch_config_list_t" (pointer . + "notmuch_config_values_t") "notmuch_config_values_t" (pointer . + "notmuch_config_pairs_t") "notmuch_config_pairs_t" (pointer . + "notmuch_indexopts_t") "notmuch_indexopts_t" "notmuch_compact_status_cb_t" (pointer . "notmuch_message_properties_t") "notmuch_message_properties_t")) (export ffi-notmuch-types) diff --git a/scratch/semgrep/requirements.txt b/scratch/semgrep/requirements.txt new file mode 100644 index 0000000..3e493ea --- /dev/null +++ b/scratch/semgrep/requirements.txt @@ -0,0 +1,2 @@ +sentence-transformers +chromadb diff --git a/scratch/semgrep/semantic-search.el b/scratch/semgrep/semantic-search.el new file mode 100644 index 0000000..e1692d0 --- /dev/null +++ b/scratch/semgrep/semantic-search.el @@ -0,0 +1,148 @@ +;;; semantic-search.el --- Search for semantic similarity of text -*- lexical-binding: t; -*- +;; +;; Copyright (C) 2024 Óscar Nájera +;; +;; Author: Óscar Nájera <hi@oscarnajera.com> +;; Maintainer: Óscar Nájera <hi@oscarnajera.com> +;; Created: February 04, 2024 +;; Modified: February 04, 2024 +;; Version: 0.1.0 +;; Keywords: convenience data docs files hypermedia i18n matching tools +;; Homepage: https://github.com/titan/semantic-search +;; Package-Requires: ((emacs "27.1")) +;; +;; This file is not part of GNU Emacs. +;; +;;; Commentary: +;; +;; Search for semantic similarity of documents at a paragraph level +;; +;;; Code: + + +(require 'url) +(require 'org-element) +(require 'org-roam-db) +(require 'dash) + +;; Silence byte-compiler. +(defvar url-http-end-of-headers) + +(defcustom semantic-search-server-url "http://localhost:8080" + "Address where the Chromadb server is listening." + :type 'url + :group 'semantic-search) + +(defun semantic-search--connect (method data) + "Synchronous query to the server." + (let ((url-request-method "POST") + (url-request-extra-headers '(("Content-Type" . "application/json"))) + (url-request-data (encode-coding-string + (json-serialize `(,method ,data)) + 'utf-8))) + (with-current-buffer + (url-retrieve-synchronously semantic-search-server-url) + (goto-char url-http-end-of-headers) + (json-read)))) + +(defun semantic-search--org-id (paragraph &optional default) + (--> + (org-element-map + (org-element-property :parent paragraph) + 'node-property + (lambda (np) + (cons + (org-element-property :key np) + (org-element-property :value np)))) + (assoc 'ID it #'string=) + (cdr it) + (org-string-nw-p it) + (or it default))) + +(defun semantic-search--prepare-paragraph (file-id) + (lambda (paragraph) + (list + :document (substring-no-properties (org-element-interpret-data paragraph)) + :metadata (list :start-point + (org-element-property :begin paragraph) + :node-id + (semantic-search--org-id paragraph file-id))))) + +(defun semantic-search--add-buffer () + (interactive) + (if (eq major-mode 'org-mode) + (-some--> + (org-element-map + (org-element-parse-buffer) + 'paragraph + (semantic-search--prepare-paragraph (org-id-get (point-min) 'create))) + (cl-coerce it 'vector) + ;; (json-serialize it) + ;; (f-write it 'utf-8 "/tmp/out.json") + ;; (message "%S" it) + (semantic-search--connect :insert it)) + (user-error "This only works on org-mode"))) + +(defun semantic-search--roam-data (entries) + (thread-last + (cl-mapcar (lambda (meta) + (alist-get 'node-id meta)) + entries) + (delete-dups) + (vconcat) + (org-roam-db-query [:select [id title file] + :from nodes + :where (in id $v1)]))) + +(defun semantic-search--del-buffer (org-ids) + (interactive (list (org-id-get))) + (unless (null org-ids) + (semantic-search--connect :delete org-ids))) + +(defun semantic-search-pick-org-element () + (when-let ((context (ignore-errors (org-element-context)))) + (filter-buffer-substring (org-element-property :begin context) + (org-element-property :end context)))) + +(defun semantic-search--sync-db () + (org-roam-dolist-with-progress (file (nreverse (org-roam-list-files))) + "importing to semantic search" + (org-roam-with-file file nil + (semantic-search--add-buffer)))) + +;; (semantic-search--sync-db) +(defun semantic-search (text) + (interactive (list (or (semantic-search-pick-org-element) + (read-from-minibuffer "What are you looking for? ")))) + (-let (((&alist 'distances 'documents 'metadatas) + (semantic-search--connect :query text))) + (with-current-buffer (get-buffer-create "*Semantic Search*") + (erase-buffer) + (org-mode) + (insert "#+title: Looking for:\n" text "\n") + (cl-mapc + (lambda (entry-distances entry-document entry-metadatas) + (let ((data (semantic-search--roam-data entry-metadatas))) + (cl-mapc + (lambda (d paragraph meta) + (unless (zerop d) + (-let* ((node-id (alist-get 'node-id meta)) + ((_ title file) (assoc node-id data #'string=)) + (pos + (if file + (with-temp-buffer + (insert-file-contents file) + (line-number-at-pos (or (alist-get 'start-point meta) 1))) + 1))) + (insert + (format "* [[file:%s::%d][%s]]\n" file pos title) + "- Distance :: " (number-to-string d) "\n" + paragraph ?\n) + (org-fill-paragraph)))) + entry-distances entry-document entry-metadatas))) + distances documents metadatas) + (goto-char (point-min)) + (display-buffer (current-buffer))))) + +(provide 'semantic-search) +;;; semantic-search.el ends here diff --git a/scratch/semgrep/server.py b/scratch/semgrep/server.py new file mode 100644 index 0000000..becabbb --- /dev/null +++ b/scratch/semgrep/server.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +import argparse +import collections +import hashlib +import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +import chromadb + + +def checksum(string: str): + sha256 = hashlib.sha256() + sha256.update(string.encode("utf-8")) + return sha256.hexdigest()[:32] + + +def ensure_list(data): + if isinstance(data, str): + return [data] + if isinstance(data, list): + if all(isinstance(l, str) for l in data): + return data + raise ValueError("Data must be a list of strings") + + +def delete_nodes(nodes): + for node in nodes: + collection.delete(where={"node-id": node}) + + +class MyRequestHandler(BaseHTTPRequestHandler): + def do_POST(self): + content_length = int(self.headers["Content-Length"]) + post_data = self.rfile.read(content_length).decode("utf-8") + + try: + data = json.loads(post_data) + response_message = f"Received POST request with data: '{data}'\n" + self.log_message(response_message) + except ValueError: + response_message = "Invalid JSON data" + self.send_response(400) + + if query := data.get("query"): + self.log_message("Processing query '%s'", query.replace("\n", " ").strip()) + response = collection.query(query_texts=ensure_list(query)) + elif delete_set := data.get("delete"): + delete_nodes(ensure_list(delete_set)) + response = f"Deleted nodes {delete_set}" + elif paragraphs := data.get("insert"): + data, metadata = drop_duplicates(paragraphs) + nodes = set(m.get("node-id") for m in metadata) + self.log_message("Processing metadata %s", nodes) + delete_nodes(nodes) + collection.add( + documents=data, metadatas=metadata, ids=list(map(checksum, data)) + ) + response = f"Successfully inserted {nodes}" + else: + raise ValueError(f"Used wrong method. Sent: {data.keys()}") + + response_message = json.dumps(response) + + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + self.wfile.write(response_message.encode("utf-8")) + + +def run_server(port=8080): + server_address = ("", port) + httpd = HTTPServer(server_address, MyRequestHandler) + print(f"Server running on port {port}") + httpd.serve_forever() + + +def drop_duplicates(paragraphs): + data = [data["document"].replace("\n", " ").strip() for data in paragraphs] + metadata = [data["metadata"] for data in paragraphs] + dups = (x for x, count in collections.Counter(data).items() if count > 1) + to_drop = [] + for no in dups: + to_drop.extend([i for i, d in enumerate(data) if d == no][1:]) + to_drop.sort(reverse=True) + for index in to_drop: + data.pop(index) + metadata.pop(index) + return data, metadata + + +def parse_arguments(args=None): + parser = argparse.ArgumentParser( + description="Run Semantic database server", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-db", "--database", default="org-roam", help="Name of the collection" + ) + parser.add_argument( + "-D", + "--database-dir", + default="semantic-roam", + help="Directory where to store database files", + ) + parser.add_argument( + "-p", "--port", default=8080, type=int, help="Port where server listens" + ) + + return parser.parse_args(args) + + +if __name__ == "__main__": + args = parse_arguments() + client = chromadb.PersistentClient(path=args.database_dir) + collection = client.get_or_create_collection(args.database) + run_server(args.port) diff --git a/scratch/semgrep/tests.py b/scratch/semgrep/tests.py new file mode 100644 index 0000000..0e5d233 --- /dev/null +++ b/scratch/semgrep/tests.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +import server as s + + +def test_drop_dup(): + sample = [ + {"document": "Hello", "metadata": 5}, + {"document": "World", "metadata": 8}, + {"document": "Hello", "metadata": 6}, + {"document": "Good", "metadata": 3}, + {"document": "World", "metadata": 9}, + {"document": "World\n\n", "metadata": 9}, + ] + + assert s.drop_duplicates(sample) == (["Hello", "World", "Good"], [5, 8, 3]) diff --git a/scratch/semgrep/utils.py b/scratch/semgrep/utils.py new file mode 100644 index 0000000..66ee410 --- /dev/null +++ b/scratch/semgrep/utils.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# This functions don't necessarily stand alone. I used on debug process +import uuid + +response = collection.query( + query_texts="machines scale", + where={"node-id": "496d4874-be24-4601-8a87-214d55e11297"}, +) +collection.query(query_texts="machines scale") + + +def get_node(node_id): + return collection.get(where={"node-id": node_id}) + + +def get_data(amount): + nodeid = str(uuid.uuid4()) + data = ["heelo" for _ in range(amount)] + metadata = [{"node-id": nodeid, "point": i} for i in range(amount)] + ids = [str(uuid.uuid4()) for _ in range(amount)] + return {"documents": data, "metadatas": metadata, "ids": ids} |