fot-sloan-companion
Fabien Benetou 3 weeks ago
parent 07888d0963
commit cc36b10d50
  1. 60
      Dockerfile
  2. 547
      companion.js
  3. 3
      get_thumbnails
  4. 1638
      index.html
  5. 930
      jxr-core_branch_teleport.js
  6. 8
      package.json
  7. 14
      packing_directory_script
  8. 194
      pmwiki_reader.lua
  9. 26
      withdefault.jxrstyles.json

@ -0,0 +1,60 @@
from node:20-bookworm
# probably a bad start here as a lot of packages are large so no benefit
# could restarting here from Debian instead
# for now only tested those but theoretically using the same software it all should work
# .odg .pdf .mov .svg
# new ones thanks to Debian
# .blend
RUN apt update && apt -y upgrade
RUN apt install -y rclone # tested for DropBox
RUN apt install -y ghostscript # tested for .pdf via convert
RUN apt install -y imagemagick # tested for .jpg and .pdf
RUN apt install -y libreoffice # tested for .odp
RUN apt install -y default-jre # might be needed for soffice
#RUN apt install -y openjdk8-jre # might be needed for soffice
RUN apt install -y ffmpeg # tested for .mov
RUN apt install -y sox # tested for .wav (not even sure we use over ffmpeg though... but it's in there)
RUN apt install -y inkscape # tested for .svg
# pointless without texlive unfortunately
# RUN apk add pandoc # tested with .epub and .pmwiki (via lua filter)
# WARNING, this makes the image HUGE, from 2GB or less to 6GB, pandoc itself is fine but texlive-full is massage
# RUN apk add texlive-full # needed for pandoc
# RUN apk add chromium # untested, needed for rendering HTML
# RUN npx puppeteer browsers install chrome # not enough
# seems particularly problematic on Alpine
# https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-on-alpine
RUN wget https://ftp.nluug.nl/pub/graphics/blender/release/Blender4.2/blender-4.2.3-linux-x64.tar.xz
RUN tar -xf blender*.tar.xz
RUN ln -s /blender*/blender /usr/bin/blender
# RUN apt install -y blender # untested
# here v 3.x whereas locally v4.x
# Segmentation fault (core dumped)
# RUN apt install -y pipx
# RUN pipx install rmc # untested for .rm
# does not add to the path, available as /root/.local/bin/rmc
WORKDIR /usr/app
COPY ./ /usr/app
COPY ./rclone.conf /root/.config/rclone/rclone.conf
# surprising slow step ?!
RUN npm install
# for now cheating with ./node_modules already there
EXPOSE 3000
# Set up a default command
CMD [ "node","." ]
# to test faster
# docker exec -it $(docker ps | grep companion:latest | sed "s/ .*//") sh
# then copy files from the test_files directory to public/
# should keep different version, this is huge with texlive-full
# environment variable should help probe what is available vs not available

@ -0,0 +1,547 @@
/*
* start http/https server
* on update, e.g. adding removing file, broadcast as SSE (as defined at the end)
* https://nodejs.org/docs/latest/api/fs.html#fswatchfilename-options-listener
* try to send MIME types
* could use https://iwearshorts.com/blog/serving-correct-mimes-with-node-and-express/
* yet theoretically ContentType might be enough https://www.geeksforgeeks.org/difference-between-contenttype-and-mimetype/
* examples with .txt and .json
*/
/* potential improvements
try to get locally generated thumbnails
execSync( './get_thumbnails' ) // takes less than a second
// but still should NOT be done every single time a file is added, otherwise directory will trigger a LOT of such potentially copying
ramfs to read/write to files yet faster
https://www.linuxquestions.org/questions/linux-general-1/using-inotify-with-ramfs-672764/
search
as files get added and potentially converted
their content
specifically text at first
can be extended on png/pdf for OCR
tesseract filename.jpg out -l eng
when NOT coming from another format that already provides text
e.g PDF
should be indexed to provide search capability too
can start with a single (JSON) datastructure
filename, textContent
per user scoping
could prefix most routes with a username (and hash for pseudo privacy)
sshkeys per user
allowing to bring content to other devices
need to be accessible though
Tailscale?
trust issue
fine if self-hosted...
zotero JSON
biblio management
reMarkable highlights
full loop, read already, continue your work
area/volumes
outcome should be ideally space related too
not send
could drag&drop duplicate
drag&drop virtual reMarkable on file instead
keep the spatial aspect
could get from reMarkable, e.g latest sketch
pull from, a la PDF current cloning page
kanban tagging
to read
to share
...
highlighting back, cf https://x.com/utopiah/status/1847620072090595547
should get
stamped PDF with JSON of highlights
stamped PDF
pdftk augmented_paper.pdf burst
convert highlight.png highlight.pdf
pdftk pg_0012.pdf stamp highlight.pdf output test.pdf
mv test.pdf pg_0012.pdf
pdftk pg_00*pdf output stamped.pdf
JSON of highlight
see ~/Prototypes/pdf_highlight_from_position/ for a way to go from x,y coordinates to text
using PDF.js-extract in NodeJS
this is a very large dependency, ~235MB, due to node-canvas (181MB)
could do gradual window growth
start with exact line
if fail, try N pixel above/below
if tail try again with 2*N, repeat
consider https://www.w3.org/TR/annotation-model/ as way to save and share
consider public facing version
could rely on WebDAV, cf https://webdav.benetou.fr
with an upload Web interface
each session would be "private" thanks to a generated keyword, e.g. banana
by default the user would never have to type it, yet it could be used to restore a past session
note though that probably quite a few format conversion will break by being truly headless
need to be tested
yet quite a few starting with PDFs, thanks to Pandoc alone, should work
Telegram bot / Slack bot as alternative entry
fs itself, with subdirectory, as manipulable entity
e.g. using https://github.com/mihneadb/node-directory-tree
npx directory-tree -p public/ --attributes type,extension,size -o public/filesystem.json
relying on thumbnails already generated locally
e.g. ~/.cache/thumbnails
described in https://askubuntu.com/questions/1084640/where-are-the-thumbnails-of-a-new-image-located
based on MD5 sum of full path
currently supporting
pdf blend html png jpg mp4
not supporting
glb json txt gz zip
rare or unofficial
pmwiki canvas
custom
entity component
additional parseable data
audio or video to text (ideally JSON)
ffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav
cd ~/Prototypes/whisper.cc && ./main -m /home/fabien/Prototypes/whisper.cpp/models/ggml-base.en.bin -f samples/jfk.wav -ojf ./res
getting a samples/jfk.wav.json as output
PDF to text (for highlights)
using PDF.js-extract in NodeJS
PDF images to images
pdfimages
no position, only number and page number
seems that despite -j (supposedly forcing JPEG conversion) leaves some files unchanged
e.g. .ppm from augmented_paper.pdf
pdfimages -j -p au.pdf au/image
which would in turns need convert all .ppm to jpg in that directory
could be partial .json of the result, thus supporting subdirectory
au.pdf.images.json
viewer can check if this file is present, if so use it too
image to text (OCR)
tesseract (or more modern alternatives, but require relatively complex setup)
for custom made types consider updating ~/.config/mimeapps.list
extending to supporting materials, not "just" PDF
https://x.com/utopiah/status/1851581594340925555
export endpoint
saving layout back
possibly with, e.g. by URL parameter, layout loader
itself could be listed as a component so that all present layouts can be swapped per user
node email, cf Telegram work done in the past
/home/fabien/fabien/Prototypes/nodemail/index.js
*/
const express = require('express')
const https = require('https')
const fs = require('fs')
const ip = require('ip')
const app = express()
const port = 3000
const {execSync} = require('child_process')
const nodeHtmlToImage = require('node-html-to-image')
const converters = ['convert', 'soffice', 'inkscape', 'blender ', 'pandoc ', 'ffmpeg', '~/Apps/rmc/bin/rmc ' ]
// should check presence and enable/disalbed conversion based on them, if not present provide hints on how to install
// currently crashes if not present
// consider the distributed fashion i.e. https://git.benetou.fr/utopiah/offline-octopus/src/branch/master/index.js#L84
app.get('/files', (req, res) => {
res.json( fs.readdirSync('public') )
})
app.get('/', (req, res) => {
res.send('')
})
/*
SSE minimal client
used only for redirections, e.g. redirecting on public/filename.pdf
no 2D viewer, etc
could optional we used for live debugging during demo
consider the equivalent for drag&drop of file and their content
namely allowing the upload of files
arguably not needed with e.g. DropBox
but could be more direct if handled without 3rd party
removing the need for any installation yet still reacting, a la reMarkable file drop
could filter by name
e.g. device or person
so that updates are only received by a specific kind of devices
/sseclient/clientname
could also rely on <input>
should display visual update, not just CLI
*/
app.get('/sseclient', (req, res) => {
res.send(sse_html)
})
const sse_html = `
<!DOCTYPE html>
<html lang="en">
<body></body>
<script>
const source = new EventSource('/events');
source.addEventListener('message', message => {
let data = JSON.parse( message.data )
console.log(data)
if (data && data.open) {
let li = document.createElement('li')
let a = document.createElement('a')
a.href = data.open
a.innerText = data.open
document.body.appendChild( li )
li.appendChild( a )
window.open(data.open, '_blank')
}
})
</script>
</html>
`
app.get('/remoteredirect/:filename', (req, res) => {
let filename = req.params.filename
let data = {}
data.open = '/'+filename
sendEventsToAll(data)
res.json(data.open)
})
app.use(express.static('public'))
const privateKey = fs.readFileSync("privatekey.pem", "utf8");
const certificate = fs.readFileSync("certificate.pem", "utf8");
const credentials = { key: privateKey, cert: certificate };
const webServer = https.createServer(credentials, app);
webServer.listen(port, () => {
console.log(`open https://${ip.address()}:${port}/index.html on your WebXR device on the same network`)
});
// see HTML conversion example, cf ~/Prototypes/fot_sloan_companion_with_HTML
// surprisingly does not grow the size much, and even works with WebGL, so maybe relying on Chromium already installed
// probably does not work so well headlessly
// failed via container, cf fot_rpi5/Dockerfile
// Error: Unable to launch browser, error message: Failed to launch the browser process! spawn /root/.cache/puppeteer/chrome-headless-shell/linux-128.0.6613.119/chrome-headless-shell-linux64/chrome-headless-shell ENOENT
// from https://git.benetou.fr/utopiah/offline-octopus/src/branch/master/index.js
// SSE from https://www.digitalocean.com/community/tutorials/nodejs-server-sent-events-build-realtime-app
// adapted from jxr-permanence
let clients = [];
function eventsHandler(request, response, next) {
const headers = {
'Content-Type': 'text/event-stream',
'Connection': 'keep-alive',
'Cache-Control': 'no-cache'
};
response.writeHead(200, headers);
const clientId = Date.now();
const newClient = { id: clientId, response };
clients.push(newClient);
request.on('close', () => { clients = clients.filter(client => client.id !== clientId); });
}
function sendEventsToAll(data) {
// function used to broadcast
clients.forEach(client => client.response.write(`data: ${JSON.stringify(data)}\n\n`))
}
let savedLayout
const rmDirectory = 'remarkablepro'
app.get('/send-remarkablepro/:filename', (req, res) => {
filename = req.params.filename
if (filename.includes('/')) {
res.json('error, invalide filename')
} else {
// same paradigm i.e. a directory per drmDirectory+'/'+filenameevice, with automated conversion to the supported target format
let src = 'public'+'/'+filename
let dest = rmDirectory+'/'+filename
fs.copyFile(src, dest, (err) => {
// if (err) throw err;
console.log(src,'was copied to',dest);
});
res.json(filename)
}
})
app.get('/save-layout/:layout', (req, res) => {
savedLayout = req.params.layout
// unsafe, assume JSON but could be anything
try{ JSON.parse(savedLayout) } catch { console.log('not json, file NOT saved!'); res.json('failed saved, not proper JSON!'); return }
console.log('savedLayout', savedLayout)
// could be saved to disk, thus to file, too
let savedFilename = Date.now()+'.layout.json'
fs.writeFileSync('./public/'+savedFilename, savedLayout)
// might be better to save in a dedicated directory in ./public
res.json(savedFilename)
})
app.get('/export-email', (req, res) => {
if (!savedLayout){
res.json('layout missing, email NOT sent')
return
}
let content = JSON.stringify(savedLayout)
let title = "your Spatial Scaffolding companion based layout saved"
const emailData = {
Recipients: [ { Email: "fabien@iterative-explorations.com", Fields: { name: "fabien" } } ],
Content: {
Body: [
{ ContentType: "HTML", Charset: "utf-8", Content: '\n\n' + content + "\n sent to {name} \n" }
],
From: "noreplay@mymatrix.ovh", Subject: "email via node: " + title
}
};
emailsApi.emailsPost(emailData, callback); //not needed here
res.json('email sent')
})
app.get('/events', eventsHandler);
// for example /events.html shows when /scan begings (but not ends)
let newFiles = []
fs.watch('public', (eventType, filename) => {
console.log(`event type is: ${eventType}`); // rename can also be deleting...
// could consequently check if the file still exists, if not, had been deleted
if (filename) {
sendEventsToAll({filename,eventType})
console.log(`filename provided: ${filename}`)
if (eventType == "rename"){
if (!fs.existsSync(filename)) {
console.log(`${filename} deleted`)
}
}
if (eventType == "change"){
if (newFiles.includes(filename)){
console.log( 'skip, not a new file')
} else {
console.log('new file', filename, '_________________________________________')
if ( !filename.includes('.live') ) {
newFiles.push(filename)
// bypass on convention, e.g. live in the filename
// alternatively could be a dedicated subdirectory
} else { console.log('live file, no future ignoring') }
// all those should be within a try/catch block as they can fail for many reasons
if (filename.endsWith('.pdf')) execSync( 'convert "'+filename+'" "'+filename+'.jpg"', {cwd:'public'})
// if (filename.endsWith('.pdf')) execSync( 'convert -density 600 '+filename+' -background white -flatten -resize 25% '+filename+'.jpg', {cwd:'public'})
// untested, high res
if (filename.endsWith('.ods')) execSync( 'soffice --headless --convert-to jpg '+filename, {cwd:'public'})
// .xls also works
if (filename.endsWith('.odg')) execSync( 'soffice --headless --convert-to jpg '+filename, {cwd:'public'})
if (filename.endsWith('.odp')) execSync( 'soffice --headless --convert-to pdf '+filename, {cwd:'public'})
// automatically "cascade" to PDF conversion
if (filename.endsWith('.svg')) execSync( 'inkscape --export-type="png" '+filename, {cwd:'public'})
// execSync( 'inkscape --export-type="png" '+filename+'; convert '+filename.replace('svg','png')+' '+filename.replace('svg','png'), {cwd:'public'})
// could probe to see if the commands, e.g. convert, inkscape, etc are available
// if not, return a warning, suggesting to install them
// (could try using the local package manager)
if (filename.endsWith('.blend')) execSync( `blender "${filename}" -b --python-expr "import bpy;bpy.ops.export_scene.gltf( filepath='test.glb', export_format='GLB', use_active_collection =True)"`, {cwd:'public'})
if (filename != 'index.html' && filename.endsWith('.html')) {
// could potentially be done via Pandoc too
let data = fs.readFileSync('./public/'+filename, { encoding: 'utf8', flag: 'r' });
nodeHtmlToImage({ output: './public/'+filename+'.png', html: data }).then(() => console.log('The image was created successfully!'))
}
if (filename.endsWith('.aframe.component')) console.log('aframe component, to live reload')
if (filename.endsWith('.aframe.entity')) console.log('aframe entity, to live reload')
// nothing to do serve side though, see client side
if (filename.endsWith('.epub')) execSync( 'pandoc '+filename+" -o "+filename+".pdf", {cwd:'public'})
// pandoc allows quite few more formats, e.g. docx, ODT, RTF but also MediaWiki markup, Markdown, etc even reveal.js slides
// interestingly also for this work, BibTeX and CSL JSON, and other bibliographic formats
// e.g. pandoc biblio.bib -t csljson -o biblio2.json from https://pandoc.org/demos.html
if (filename.endsWith('.rm')) execSync( '~/Apps/rmc/bin/rmc -t svg -o '+filename+'.svg '+filename, {cwd:'public'})
// see also latestRemarkableNoteToXR
// automatically "cascade" to SVG conversion
if (filename.endsWith('.wav')) execSync( 'ffmpeg -i '+filename+" -y "+filename+".mp3", {cwd:'public'})
if (filename.endsWith('.mov')) execSync( 'ffmpeg -i '+filename+" -y "+filename+".mp4", {cwd:'public'})
if (filename.endsWith('.pmwiki')) execSync( 'cat '+filename+' | grep -a "^text=" | sed "s/^text=//" | sed "s/%0a/\\n/g" | sed "s/%25/%/g" | sed "s/%3c/</g" | pandoc -f ../pmwiki_reader.lua -o '+filename+".pdf", {cwd:'public'})
// untested
// requires pandoc lua filter
// automatically "cascade" to PDF conversion
// unfortunately PmWiki does not have its own filename so have to do it manually i.e. .pmwiki
// cf https://github.com/tfager/pandoc-pmwiki-reader/issues/1
// cat Fabien.Principle.pmwiki | grep -a "^text=" | sed "s/^text=//" | sed "s/"%0a"/\n/g" | sed "s/%25/%/g" | sed "s/%3c/</g" | pandoc -f pmwiki_reader.lua -o Fabien.Principle.pmwiki.pdf
}
}
} else {
console.log('filename not provided');
}
});
/* ========================= reMarkable ========================= */
let newFilesRM = []
fs.watch(rmDirectory, (eventType, filename) => {
if (filename) {
if (eventType == "rename"){ if (!fs.existsSync(filename)) { console.log(`${filename} deleted`) } }
if (eventType == "change"){
if (newFilesRM.includes(filename)){
console.log( 'skip, not a new file')
} else {
console.log('new file', filename, '_________________________________________')
if ( !filename.includes('.live') ) {
newFilesRM.push(filename)
} else { console.log('live file, no future ignoring') }
if (!filename.endsWith('.pdf') && !filename.endsWith('epub')){
console.log('this target only supports pdf and epub directory for now')
// could instead here try conversion
return
} else {
// let sendRmCmd = 'scp "'+filename+'" remarkable2:/home/root/ && ssh remarkable2 -t "source /home/root/.bashrc; addWithMetadataIfNewSpaScaDir '+filename+'; systemctl restart xochitl"'
let sendRmCmd = 'scp "'+filename+'" remarkablepro:/home/root/ && ssh remarkablepro -t "source /home/root/remarkable_functions.sh; addWithMetadataIfNew '+filename+'; systemctl restart xochitl"'
// could improve using krop via cli
// krop --go --trim filename.pdf -o filename-cropped.pdf
console.log(sendRmCmd) // verification
// assuming the right ssh key and parameters (usually in ~/.ssh/ for the current user running the companion)
// does not work on containerized environment (lacking such access)
// could be considered with offline-octopus proper
// should be within a try/catch block as they can fail for many reasons
execSync( sendRmCmd, {cwd:rmDirectory})
}
}
}
} else {
console.log('filename not provided');
}
});
/* ========================= rclone ========================= */
const { spawn } = require('node:child_process');
const changenotify = spawn('rclone', ['test', 'changenotify', 'dropbox:']);
// testable via https://www.dropbox.com/request/TVNfsrMpTr1RcsuNisIX
//const changenotify = spawn('rclone', ['test', 'changenotify', 'googledrive:']);
// sadly didn't seem to work, account deleted and rclone config removed remote
changenotify.stdout.on('data', (data) => {
console.log(`stdout: ${data}`);
});
let newFiles_dropbox = []
changenotify.stderr.on('data', (data) => {
console.error(`stderr: ${data}`);
// console.log(data.toString()); return
// TODO ... for now stick to single file upload, otherwise with the current DropBox messaging it's a mess
// bit of a mess really... no proper timestamp, multiple messages with same file, no event type...
// stderr: 2024/10/29 23:06:24 NOTICE: "remote_directory_test/Fabien Benetou - freezer.glb": 1
if (data.toString().includes('polling every')) return
// need to sync first THEN push
let syncOutput = execSync( 'rclone copy dropbox:remote_directory_test dropbox_remote_upload/' ).toString()
// not usable output
// console.log('sync output:', syncOutput)
// probably syncing way too much, syncing on each message isn't necessary!
// should only sync on genuinely new files
let lines = data.toString().split('\n')
lines.map( (l,i) => {
let newfile = l
.replace(/.*NOTICE: "/,'')
.replace(/".*/,'')
.replace('remote_directory_test','dropbox_remote_upload')
.replace('\n','')
// nearly always 2 output
if (newfile.length && i > 0) {
console.log('--------------newfile (line ',i,'): ', newfile)
// sometimes one lines, sometimes 2...
if (!newFiles_dropbox.includes(newfile)){
if (newfile.includes(' - ')){ // DropBox heuristic...
let matches = newfile.match(/(.*)\/(.*) - (.*)\.(.*)/)
if (matches.length){
let [full,path,start,end,ext] = matches
let flipped = path+'/'+end+' '+start+'.'+ext
// assuming always with an extension for now, no directory upload with subdirectories
console.log('>>> might also exist flipped so adding it flipped:', flipped)
if (newFiles_dropbox.includes(flipped)){
console.log('>>> flipped already present! Should skip too')
}
newFiles_dropbox.push(flipped)
}
}
newFiles_dropbox.push(newfile)
console.log( 'new file, actually do sth, i.e copy : ', newfile )
// nearly file... but sometimes still a duplicate file goes through due to username added before OR after (?!)
// 2024/11/03 02:42:55 NOTICE: "remote_directory_test/Fabien Benetou - remarks_nlnet.txt": 1
// 2024/11/03 02:42:55 NOTICE: "remote_directory_test/remarks_nlnet Fabien Benetou.txt": 1
// here on local filesystem we only get 1... but sometimes we get both! (?!)
// ... so we should check includes without username (regardless of position or hypthen)
// could ignore if includes but for that need to know what is the filename vs username in a reliable way
try {
//fs.copyFile('./'+newfile, './public/'+newfile.split(' ').at(-1), (err) => {
let src = './'+newfile
// problematic when done at the same time so switching to kind of uuid
//let dest = './public/'+Date.now()+'.'+newfile.split('.').at(-1)
let pseudouuid = (new Date()).getTime().toString(36) + Math.random().toString(36).slice(2)
// TODO not a good solution, getting plenty of duplicates
// hopefully partly pruned now
// somehow getting 2 files for 1 transfer, not good!
// good do checksum if needed but a bit time consuming for large files
// let dest = './public/dropbox_'+pseudouuid+'.'+newfile.split('.').at(-1)
let dest = './public/dropbox_'+newfile.replace('dropbox_remote_upload/','')
fs.copyFile(src, dest, (err) => {
// if (err) throw err;
console.log(src,'was copied to',dest);
});
} catch (e) {
console.log('error copy', e)
}
} else {
console.log( 'ignoring, already present' )
}
}
})
});
changenotify.on('close', (code) => {
console.log(`child process exited with code ${code}`);
});
/*
rclone
rclone copy public/ dropbox:fot_sloan_companion_public
rclone bisync -n public/ dropbox:fot_sloan_companion_public --resync
rclone bisync -n public/ dropbox:fot_sloan_companion_public
watch -n 10 rclone bisync dropbox_remote_upload/ dropbox:remote_directory_test --resync
probably better not to use --resync for faster/lighter results
might not even want bisync here as it's always getting new content, copy is probably better
rclone copy dropbox:remote_directory_test dropbox_remote_upload/
could consider leaving that in the background running
child_process.spawn with detached option
https://nodejs.org/api/child_process.html#child_process_child_process_spawn_command_args_options
changenotify https://rclone.org/commands/rclone_test_changenotify/
rclone test changenotify dropbox:
says polling every 10s but seems much faster
*/
/* ========================= elasticemail ========================= */
var ElasticEmail = require('@elasticemail/elasticemail-client');
var defaultClient = ElasticEmail.ApiClient.instance;
var apikey = defaultClient.authentications['apikey'];
apikey.apiKey = "0C3D85070303586EB6A3C74E770942F903ACA0C46AFEEDB86CA334A8937056CFFDE92AE7D109FF5AAC41AB2B3CCFF1EB"
const emailsApi = new ElasticEmail.EmailsApi();
const callback = (error, data, response) => {
if (error) {
console.error(error);
} else {
console.log('API called successfully. Email sent.');
}
};

@ -0,0 +1,3 @@
find_thumbnail(){ full_path="file://$(realpath -s "$1")"; md5name=$(printf %s "${full_path// /%20}" | md5sum); find ~/.cache/thumbnails/ -name "${md5name%% *}.png"; }
cd public
for f in *; do echo -n "$f " ; cp "$(find_thumbnail $f | grep large)" "thumbnails/$f.png"; done;

File diff suppressed because it is too large Load Diff

@ -0,0 +1,930 @@
const prefix = /^jxr /
const codeFontColor = "lightgrey"
const fontColor= "white"
var selectedElement = null;
var targets = []
const zeroVector3 = new THREE.Vector3()
var bbox = new THREE.Box3()
bbox.min.copy( zeroVector3 )
bbox.max.copy( zeroVector3 )
var selectionBox = new THREE.BoxHelper( bbox.object3D, 0x0000ff);
var groupHelpers = []
var primaryPinchStarted = false
var wristShortcut = "jxr switchToWireframe()"
var selectionPinchMode = false
var groupingMode = false
var hudTextEl // should instead rely on the #typinghud selector in most cases
const startingText = "[]"
var added = []
const maxItemsFromSources = 20
let alphabet = ['abcdefghijklmnopqrstuvwxyz', '0123456789', '<>'];
var commandhistory = []
var groupSelection = []
var primarySide = 0
const sides = ["right", "left"]
var pinches = [] // position, timestamp, primary vs secondary
var dl2p = null // from distanceLastTwoPinches
var selectedElements = [];
// ==================================== picking ======================================================
AFRAME.registerComponent('target', {
init: function () {
targets.push( this.el )
this.el.classList.add("collidable")
}
})
function getClosestTargetElements( pos, threshold=0.05 ){
// assumes pos has now no offset
// TODO Bbox intersects rather than position
return targets.filter( e => e.getAttribute("visible") == true)
.map( t => {
let posTarget = new THREE.Vector3()
t.object3D.getWorldPosition( posTarget )
let d = pos.distanceTo( posTarget )
return { el: t, dist : d }
})
.filter( t => t.dist < threshold && t.dist > 0 )
.sort( (a,b) => a.dist > b.dist)
}
function getClosestTargetElement( pos, threshold=0.05 ){ // 10x lower threshold for flight mode
var res = null
// assumes both hands have the same (single) parent, if any
let parentPos = document.getElementById('rig').getAttribute('position')
pos.add( parentPos )
console.log( "from getClosestTargetElements, pos:", pos ) // relative pos, should thus remove rig position, even though it makes assumptions
const matches = getClosestTargetElements( pos, threshold)
if (matches.length > 0) res = matches[0].el
return res
}
// ==================================== HUD ======================================================
var keyboardInputTarget = 'hud'
AFRAME.registerComponent('hud', {
init: function(){
var feedbackHUDel= document.createElement("a-troika-text")
feedbackHUDel.id = "feedbackhud"
feedbackHUDel.setAttribute("value", "")
feedbackHUDel.setAttribute("position", "-0.05 0.01 -0.2")
feedbackHUDel.setAttribute("scale", "0.05 0.05 0.05")
this.el.appendChild( feedbackHUDel )
var typingHUDel = document.createElement("a-troika-text")
typingHUDel.id = "typinghud"
typingHUDel.setAttribute("value", startingText)
typingHUDel.setAttribute("position", "-0.05 0 -0.2")
typingHUDel.setAttribute("scale", "0.05 0.05 0.05")
this.el.appendChild( typingHUDel )
hudTextEl = typingHUDel // should rely on the id based selector now
document.addEventListener('keyup', function(event) {
if (keyboardInputTarget != 'hud') return
parseKeys('keyup', event.key)
});
document.addEventListener('keydown', function(event) {
if (keyboardInputTarget != 'hud') return
parseKeys('keydown', event.key)
});
}
})
function appendToFeedbackHUD(txt){
setFeedbackHUD( document.querySelector("#feedbackhud").getAttribute("value") + " " + txt )
}
function setFeedbackHUD(txt){
document.querySelector("#feedbackhud").setAttribute("value",txt)
setTimeout( _ => document.querySelector("#feedbackhud").setAttribute("value","") , 2000)
}
function appendToHUD(txt){
const textHUD = document.querySelector("#typinghud").getAttribute("value")
if ( textHUD == startingText)
setHUD( txt )
else
setHUD( textHUD + txt )
}
function setHUD(txt){
document.querySelector("#typinghud").setAttribute("value",txt)
}
function showhistory(){
setFeedbackHUD("history :\n")
commandhistory.map( i => appendToHUD(i.uninterpreted+"\n") )
}
function saveHistoryAsCompoundSnippet(){
addNewNote( commandhistory.map( e => e.uninterpreted ).join("\n") )
}
// ==================================== pinch primary and secondary ======================================================
AFRAME.registerComponent('pinchsecondary', {
init: function () {
this.el.addEventListener('pinchended', function (event) {
selectedElement = getClosestTargetElement( event.detail.position )
selectedElements.push({element:selectedElement, timestamp:Date.now(), primary:false})
// if close enough to a target among a list of potential targets, unselect previous target then select new
if (selectedElement) interpretJXR( selectedElement.getAttribute("value") )
selectedElement = null
});
this.el.addEventListener('pinchmoved', function (event) {
if (selectionPinchMode){
bbox.min.copy( event.detail.position )
setFeedbackHUD( "selectionPinchMode updated min")
if (!bbox.max.equal(zeroVector3))
selectionBox.update();
}
});
this.el.addEventListener('pinchstarted', function (event) {
if (!selectionPinchMode) bbox.min.copy( zeroVector3 )
if (selectionPinchMode) setFeedbackHUD( "selectionPinchMode started")
});
},
remove: function() {
// should remove event listeners here. Requires naming them.
}
});
AFRAME.registerComponent('pinchprimary', { // currently only 1 hand, the right one, should be switchable
// consider instead https://github.com/AdaRoseCannon/handy-work/blob/main/README-AFRAME.md for specific poses
// or https://aframe.io/aframe/examples/showcase/hand-tracking/pinchable.js
init: function () {
var el = this.el
this.el.addEventListener('pinchended', function (event) {
// if positioned close enough to a target zone, trigger action
// see own trigger-box component. Could use dedicated threejs helpers instead.
// https://github.com/Utopiah/aframe-triggerbox-component/blob/master/aframe-triggerbox-component.js#L66
// could make trigger zones visible as debug mode
let pos = event.detail.position
let parentPos = document.getElementById('rig').getAttribute('position')
pos.add( parentPos )
var closests = getClosestTargetElements( pos )
//if (closests && closests.length > 0) // avoiding self reference
// setFeedbackHUD("close enough, could stack with "+ closests[1].el.getAttribute("value") )
var dist = event.detail.position.distanceTo( document.querySelector("#box").object3D.position )
if (dist < .1){
setFeedbackHUD("close enough, replaced shortcut with "+ selectedElement.getAttribute("value") )
wristShortcut = selectedElement.getAttribute("value")
}
if (selectedElement){
let content = selectedElement.getAttribute("value")
selectedElement.emit('released', {element:selectedElement, timestamp:Date.now(), primary:true})
}
// unselect current target if any
selectedElement = null;
if ( groupingMode ) addToGroup( event.detail.position )
selectionPinchMode = false
/*
setHUD( AFRAME.utils.coordinates.stringify( bbox.min ),
AFRAME.utils.coordinates.stringify( bbox.max ) )
bbox.min.copy( zeroVector3 )
bbox.man.copy( zeroVector3 )
*/
setTimeout( _ => primaryPinchStarted = false, 200) // delay otherwise still activate on release
var newPinchPos = new THREE.Vector3()
newPinchPos.copy(event.detail.position )
pinches.push({position:newPinchPos, timestamp:Date.now(), primary:true})
dl2p = distanceLastTwoPinches()
});
this.el.addEventListener('pinchmoved', function (event) {
// move current target if any
if (selectionPinchMode){
bbox.max.copy( event.detail.position )
if (!bbox.min.equal(zeroVector3))
selectionBox.update();
}
if (selectedElement && !groupingMode) {
let pos = event.detail.position
let parentPos = document.getElementById('rig').getAttribute('position')
pos.add( parentPos )
pos.sub( selectedElements.at(-1).startingPosition )
selectedElement.setAttribute("position", pos )
document.querySelector("#rightHand").object3D.traverse( e => {
if (e.name == "ring-finger-tip"){
selectedElement.object3D.rotation.copy( e.rotation )
}
})
// rotation isn't ideal with the wrist as tend not have wrist flat as we pinch
}
if (selectedElement) selectedElement.emit("moved")
});
this.el.addEventListener('pinchstarted', function (event) {
primaryPinchStarted = true
if (!selectionPinchMode) bbox.max.copy( zeroVector3 )
//var clone = getClosestTargetElement( event.detail.position ).cloneNode()
// might want to limit cloning to unmoved element and otherwise move the cloned one
//AFRAME.scenes[0].appendChild( clone )
//targets.push( clone )
//selectedElement = clone
selectedElement = getClosestTargetElement( event.detail.position )
if (selectedElement) {
let startingPosition = new THREE.Vector3()
selectedElement.parentEl.object3D.getWorldPosition( startingPosition )
selectedElements.push({element:selectedElement, timestamp:Date.now(), startingPosition: startingPosition, primary:true})
selectedElement.emit("picked")
} else {
AFRAME.scenes[0].emit('emptypinch', {position:event.detail.position, timestamp:Date.now() })
}
// is it truly world position? See https://github.com/aframevr/aframe/issues/5182
// setFeedbackHUD( AFRAME.utils.coordinates.stringify( event.detail.position ) )
// if close enough to a target among a list of potential targets, unselect previous target then select new
});
},
remove: function() {
// should remove event listeners here. Requires naming them.
}
});
// re-registering but no error
AFRAME.registerComponent('onreleased', { // changed from ondrop to be coherent with event name
schema: {default: ""}, // type: "string" forced to avoid object type guess parsing
// could support multi
// could check if target component is already present on this.el, if not, add it as it's required
events: {
released: function (e) {
let code = this.el.getAttribute('onreleased')
// if multi, should also look for onreleased__ not just onreleased
try {
eval( code ) // should be jxr too e.g if (txt.match(prefix)) interpretJXR(txt)
// note that event details are avaible within that code as e.detail which might not be very clear
} catch (error) {
console.error(`Evaluation failed with ${error}`);
}
}
}
})
AFRAME.registerComponent('onpicked', {
schema: {default: ""}, // type: "string" forced to avoid object type guess parsing
// could support multi
// could check if target component is already present on this.el, if not, add it as it's required
events: {
picked: function (e) {
let code = this.el.getAttribute('onpicked')
// if multi, should also look for onreleased__ not just onreleased
try {
eval( code ) // should be jxr too e.g if (txt.match(prefix)) interpretJXR(txt)
// note that event details are avaible within that code as e.detail which might not be very clear
} catch (error) {
console.error(`Evaluation failed with ${error}`);
}
}
}
})
function onNextPrimaryPinch(callback){
// could add an optional filter, e.g only on specific ID or class
// e.g function onNextPrimaryPinch(callback, filteringSelector){}
let lastPrimary = selectedElements.filter( e => e.primary ).length
let checkForNewPinches = setInterval( _ => {
if (selectedElements.filter( e => e.primary ).length > lastPrimary){
let latest = selectedElements[selectedElements.length-1].element
if (latest) callback(latest)
clearInterval(checkForNewPinches)
}
}, 50) // relatively cheap check, filtering on small array
}
function distanceLastTwoPinches(){
let dist = null
if (pinches.length>1){
dist = pinches[pinches.length-1].position.distanceTo( pinches[pinches.length-2].position )
}
return dist
}
function groupSelectionToNewNote(){
var text = ""
groupSelection.map( grpel => {
//removeBoundingBoxToTextElement( grpel )
// somehow fails...
text += grpel.getAttribute("value") + "\n"
})
groupHelpers.map( e => e.removeFromParent() )
groupHelpers = []
groupSelection = []
addNewNote( text )
}
// ==================================== keyboard ======================================================
AFRAME.registerComponent('keyboard', {
init:function(){
let generatorName = this.attrName
const horizontaloffset = .7
const horizontalratio = 1/20
alphabet.map( (line,ln) => {
for (var i = 0; i < line.length; i++) {
var pos = i * horizontalratio - horizontaloffset
addNewNote( line[i], pos+" "+(1.6-ln*.06)+" -.4", ".1 .1 .1", null, generatorName)
}
})
}
})
function parseKeys(status, key){
var e = hudTextEl
if (status == "keyup"){
if (key == "Control"){
groupingMode = false
groupSelectionToNewNote()
}
}
if (status == "keydown"){
var txt = e.getAttribute("value")
if (txt == "[]")
e.setAttribute("value", "")
if (key == "Backspace" && txt.length)
e.setAttribute("value", txt.slice(0,-1))
if (key == "Control")
groupingMode = true
if (key == "Shift" && selectedElement)
e.setAttribute("value", selectedElement.getAttribute("value") )
else if (key == "Enter") {
if ( selectedElement ){
var clone = selectedElement.cloneNode()
clone.setAttribute("scale", "0.1 0.1 0.1") // somehow lost
AFRAME.scenes[0].appendChild( clone )
targets.push( clone )
selectedElement = clone
} else {
if (txt.match(prefix)) interpretJXR(txt)
// check if text starts with jxr, if so, also interpret it.
addNewNote(e.getAttribute("value"))
e.setAttribute("value", "")
}
} else {
// consider also event.ctrlKey and multicharacter ones, e.g shortcuts like F1, HOME, etc
if (key.length == 1)
e.setAttribute("value", e.getAttribute("value") + key )
}
}
}
// ==================================== note as text and possibly executable snippet ======================================================
function addNewNote( text, position=`-0.2 1.1 -0.1`, scale= "0.1 0.1 0.1", id=null, classes="notes", visible="true", rotation="0 0 0" ){
var newnote = document.createElement("a-troika-text")
newnote.setAttribute("anchor", "left" )
newnote.setAttribute("outline-width", "5%" )
newnote.setAttribute("outline-color", "black" )
newnote.setAttribute("visible", visible )
if (id)
newnote.id = id
else
newnote.id = "note_" + crypto.randomUUID() // not particularly descriptive but content might change later on
if (classes)
newnote.className += classes
newnote.setAttribute("side", "double" )
var userFontColor = AFRAME.utils.getUrlParameter('fontcolor')
if (userFontColor && userFontColor != "")
newnote.setAttribute("color", userFontColor )
else
newnote.setAttribute("color", fontColor )
if (text.match(prefix))
newnote.setAttribute("color", codeFontColor )
newnote.setAttribute("value", text )
//newnote.setAttribute("font", "sw-test/Roboto-msdf.json")
newnote.setAttribute("position", position)
newnote.setAttribute("rotation", rotation)
newnote.setAttribute("scale", scale)
AFRAME.scenes[0].appendChild( newnote )
targets.push(newnote)
return newnote
}
AFRAME.registerComponent('annotation', {
// consider also multiple annotation but being mindful that it might clutter significantly
schema: {
content : {type: 'string'}
},
init: function () {
addAnnotation(this.el, this.data.content)
},
update: function () {
this.el.querySelector('.annotation').setAttribute('value', this.data.content )
// assuming single annotation
},
remove: function () {
this.el.querySelector('.annotation').removeFromParent()
//Array.from( this.el.querySelectorAll('.annotation') ).map( a => a.removeFromParent() )
}
})
function addAnnotation(el, content){
// could also appear only when in close proximity or while pinching
let annotation = document.createElement( 'a-troika-text' )
annotation.classList.add( 'annotation' )
annotation.setAttribute('value', content)
annotation.setAttribute('position', '0 .1 -.1')
annotation.setAttribute('rotation', '-90 0 0')
annotation.setAttribute("anchor", "left" )
annotation.setAttribute("outline-width", "5%" )
annotation.setAttribute("outline-color", "black" )
el.appendChild(annotation)
return el
}
function interpretAny( code ){
if (!code.match(/^dxr /)) return
var newcode = code
newcode = newcode.replace("dxr ", "")
//newcode = newcode.replace(/bash ([^\s]+)/ ,`debian '$1'`) // syntax delegated server side
fetch("/command?command="+newcode).then( d => d.json() ).then( d => {
console.log( d.res )
appendToHUD( d.res ) // consider shortcut like in jxr to modify the scene directly
// res might return that said language isn't support
// commandlistlanguages could return a list of supported languages
})
}
function parseJXR( code ){
// should make reserved keywords explicit.
var newcode = code
newcode = newcode.replace("jxr ", "")
newcode = newcode.replace(/(\d)s (.*)/ ,`setTimeout( _ => { $2 }, $1*1000)`)
// qs X => document.querySelector("X")
newcode = newcode.replace(/qs ([^\s]+)/ ,`document.querySelector('$1')`)
// sa X Y => .setAttribute("X", "Y")
newcode = newcode.replace(/ sa ([^\s]+) (.*)/,`.setAttribute('$1','$2')`)
// problematic for position as they include spaces
newcode = newcode.replace(/obsv ([^\s]+)/ ,`newNoteFromObservableCell('$1')`)
// TODO
//<a-text target value="jxr observe selectedElement" position="0 1.25 -0.2" scale="0.1 0.1 0.1"></a-text>
newcode = newcode.replace(/observe ([^\s]+)/,`bindVariableValueToNewNote('$1')`)
// could proxy instead... but for now, the quick and dirty way :
// e.g qs a-sphere sa color red =>
// document.querySelector("a-sphere").setAttribute("color", "red")
newcode = newcode.replace(/lg ([^\s]+) ([^\s]+)/ ,`addGltfFromURLAsTarget('$1',$2)`)
// order matters, here we only process the 2 params if they are there, otherwise 1
newcode = newcode.replace(/lg ([^\s]+)/ ,`addGltfFromURLAsTarget('$1')`)
return newcode
}
function interpretJXR( code ){
if (!code) return
if (code.length == 1) { // special case of being a single character, thus keyboard
if (code == ">") { // Enter equivalent
content = hudTextEl.getAttribute("value")
if (Number.isFinite(Number(content))) {
loadPageRange(Number(content));
} else {
addNewNote( content )
}
setHUD("")
} else if (code == "<") { // Backspace equivalent
setHUD( hudTextEl.getAttribute("value").slice(0,-1))
} else {
appendToHUD( code )
}
}
if (!code.match(prefix)) return
var uninterpreted = code
var parseCode = ""
code.split("\n").map( lineOfCode => parseCode += parseJXR( lineOfCode ) + ";" )
// could ignore meta code e.g showhistory / saveHistoryAsCompoundSnippet
commandhistory.push( {date: +Date.now(), uninterpreted: uninterpreted, interpreted: parseCode} )
console.log( parseCode )
try {
eval( parseCode )
} catch (error) {
console.error(`Evaluation failed with ${error}`);
}
// unused keyboard shortcuts (e.g BrowserSearch) could be used too
// opt re-run it by moving the corresponding text in target volume
}
function bindVariableValueToNewNote(variableName){
// from observe jxr keyword
const idName = "bindVariableValueToNewNote"+variableName
addNewNote( variableName + ":" + eval(variableName), `-0.15 1.4 -0.1`, "0.1 0.1 0.1", idName, "observers", "true" )
// could add to the HUD instead and have a list of these
return setInterval( _ => {
const value = variableName+";"+eval(variableName)
// not ideal for DOM elements, could have shortcuts for at least a-text with properties, e.g value or position
document.getElementById(idName).setAttribute("value", value)
}, 100 )
}
AFRAME.registerComponent('gltf-jxr', {
events: {
"model-loaded": function (evt) {
this.el.object3D.traverse( n => { if (n.userData.jxr) {
console.log(n.userData)
// need to make gltf become a child of a note to be executable on pinch
// try reparenting first... otherwise var clone = this.el.cloneNode(true)
// might not be great, cf https://github.com/aframevr/aframe/issues/2425
let pos = this.el.object3D.position.clone()
let rot = this.el.object3D.rotation.clone()
this.el.remove()
let note = addNewNote( n.userData.jxr, pos, "0.1 0.1 0.1", null, "gltf-jxr-source")
let clone = this.el.cloneNode(true)
clone.setAttribute('position', '0 0 0')
clone.setAttribute('scale', '10 10 10') // assuming not scaled until now, surely wrong
// need rescaling to current scale by 1/0.1, clone.setAttribute(
clone.removeAttribute('gltf-jxr')
note.appendChild(clone)
}
})
},
},
/* example of backend code to annotate the glTF
import { NodeIO } from '@gltf-transform/core';
import { ALL_EXTENSIONS } from '@gltf-transform/extensions';
const io = new NodeIO().registerExtensions(ALL_EXTENSIONS);
const document = await io.read('PopsicleChocolate.glb');
const node = document.getRoot() // doesn't seem to work.listNodes().find((node) => node.getName() === 'RootNode');
node.setExtras({jxr: "jxr addNewNote('hi')"});
await io.write('output.glb', document);
*/
});
// ==================================== interactions beyond pinch ======================================================
AFRAME.registerComponent('wristattachsecondary',{
schema: {
target: {type: 'selector'},
},
init: function () {
var el = this.el
this.worldPosition=new THREE.Vector3();
this.skip = false
if (! this.data.target ) this.skip = true
},
tick: function () {
if (this.skip) return
// could check if it exists first, or isn't 0 0 0... might re-attach fine, to test
// somehow very far away... need to convert to local coordinate probably
// localToWorld?
(primarySide == 0) ? secondarySide = 1 : secondarySide = 0
var worldPosition=this.worldPosition;
this.el.object3D.traverse( e => { if (e.name == "wrist") {
worldPosition.copy(e.position);e.parent.updateMatrixWorld();e.parent.localToWorld(worldPosition)
rotation = e.rotation.x*180/3.14 + " " + e.rotation.y*180/3.14 + " " + e.rotation.z*180/3.14
this.data.target.setAttribute("rotation", rotation)
this.data.target.setAttribute("position",
AFRAME.utils.coordinates.stringify( worldPosition ) )
// doesnt work anymore...
//this.data.target.setAttribute("rotation", AFRAME.utils.coordinates.stringify( e.getAttribute("rotation") )
}
})
},
remove: function() {
// should remove event listeners here. Requires naming them.
}
});
function doublePinchToScale(){
let initialPositionSecondary
let initialScale
let elSecondary = document.querySelector('[pinchsecondary]')
elSecondary.addEventListener('pinchmoved', movedSecondary );
function movedSecondary(event){
if (!selectedElement) return
let scale = initialScale * initialPositionSecondary.distanceTo(event.detail.position) * 50
selectedElement.setAttribute("scale", ""+scale+" "+scale+" "+scale+" ")
}
elSecondary.addEventListener('pinchstarted', startedSecondary );
function startedSecondary(event){
initialPositionSecondary = event.detail.position.clone()
if (!selectedElement) return
initialScale = AFRAME.utils.coordinates.parse( selectedElement.getAttribute("scale") ).x
}
}
// from https://aframe.io/aframe/examples/showcase/hand-tracking/pressable.js
// modified to support teleportation via #rig
AFRAME.registerComponent('pressable', {
schema:{pressDistance:{default:0.06}},
init:function(){this.worldPosition=new THREE.Vector3();this.handEls=document.querySelectorAll('[hand-tracking-controls]');this.pressed=false;},
tick:function(){
var handEls=this.handEls;var handEl;
var distance;
for(var i=0;i<handEls.length;i++){
handEl=handEls[i];distance=this.calculateFingerDistance(handEl.components['hand-tracking-controls'].indexTipPosition);
if(distance>0 && distance<this.data.pressDistance){
if(!this.pressed){this.el.emit('pressedstarted');}
this.pressed=true;return;}
}
if(this.pressed){this.el.emit('pressedended');} // somehow happens on click, outside of VR
this.pressed=false;
},
calculateFingerDistance:function(fingerPosition){
let parentPos = document.getElementById('rig').getAttribute('position')
fingerPosition.add( parentPos )
var el=this.el;
var worldPosition=this.worldPosition;
worldPosition.copy(el.object3D.position);
el.object3D.parent.updateMatrixWorld();
el.object3D.parent.localToWorld(worldPosition);
return worldPosition.distanceTo(fingerPosition);
}
});
AFRAME.registerComponent('start-on-press', {
// should become a property of the component instead to be more flexible.
init: function(){
let el = this.el
this.el.addEventListener('pressedended', function (event) {
console.log(event)
// should ignore that if we entered XR recently
if (!primaryPinchStarted && wristShortcut.match(prefix)) interpretJXR(wristShortcut)
// seems to happen also when entering VR
// other action could possibly based on position relative to zones instead, i.e a list of bbox/functions pairs
})
}
})
function thumbToIndexPull(){
let p = document.querySelector('[pinchprimary]')
let tip = new THREE.Vector3(); // create once an reuse it
let proximal = new THREE.Vector3(); // create once an reuse it
let thumb = new THREE.Vector3(); // create once an reuse it
let touches = []
const threshold_thumb2tip = 0.01
const threshold_thumb2proximal = 0.05
let indexesTipTracking = setInterval( _ => {
// cpnsider getObjectByName() instead
p.object3D.traverse( e => { if (e.name == 'index-finger-tip' ) tip = e.position })
//index-finger-phalanx-distal
//index-finger-phalanx-intermediate
p.object3D.traverse( e => { if (e.name == 'index-finger-phalanx-proximal' ) proximal = e.position })
p.object3D.traverse( e => { if (e.name == 'thumb-tip' ) thumb = e.position })
let touch = {}
touch.date = Date.now()
touch.thumb2tip = thumb.distanceTo(tip)
if (!touch.thumb2tip) return
touch.thumb2proximal = thumb.distanceTo(proximal)
//console.log( touch.thumb2tip, touch.thumb2proximal )
// usually <1cm <4cm (!)
//if ((touch.thumb2tip && touch.thumb2tip < threshold_thumb2tip)
//|| (touch.thumb2proximal && touch.thumb2proximal < threshold_thumb2proximal))
if (touch.thumb2tip < threshold_thumb2tip
|| touch.thumb2proximal < threshold_thumb2proximal){
if (touches.length){
let previous = touches[touches.length-1]
if (touch.date - previous.date < 300){
if (touch.thumb2tip < threshold_thumb2tip &&
previous.thumb2proximal < threshold_thumb2proximal){
console.log('^')
p.emit('thumb2indexpull')
}
if (touch.thumb2proximal < threshold_thumb2proximal &&
previous.thumb2tip < threshold_thumb2tip){
console.log('v')
p.emit('thumb2indexpush')
}
}
}
touches.push(touch)
}
}, 50)
// TODO
// Bind thumb2indexpush/thumb2indexpull to zoom in/out "world" i.e all assets that aren't "special" e.g self, lights, UI
}
let changeovercheck
AFRAME.registerComponent('changeover', {
schema: { color : {type: 'string'} },
init: function () {
// (this.el, this.data.content)
if (changeovercheck) return
let player = document.getElementById('player') // assuming single player, non networked
console.log('adding timer')
changeovercheck = setInterval( _ => {
let pos = player.getAttribute('position').clone()
pos.y = 0.1 // hard coded but should be from component element
let hits = Array.from(document.querySelectorAll('[changeover]'))
.filter( e => e.getAttribute("visible") == true)
.map( t => { return { el: t, dist : pos.distanceTo(t.getAttribute("position") ) } })
.filter( t => t.dist < 0.02 )
.sort( (a,b) => a.dist > b.dist)
//console.log(hits.length)
if (hits.length>0) {
setFeedbackHUD('touching cone')
console.log('touching cone')
hits[hits.length-1].el.setAttribute('color', 'red')
}
}, 50)
}
})
// to add only on selectable elements, thus already with a target component attached
AFRAME.registerComponent('pull', {
events: {
picked: function (evt) {
this.startePos = this.el.getAttribute('position').clone()
this.starteRot = this.el.getAttribute('rotation')//.clone() not necessary as converted first
this.decimtersEl = document.createElement('a-troika-text')
AFRAME.scenes[0].appendChild(this.decimtersEl)
},
moved: function (evt) {
let pos = AFRAME.utils.coordinates.stringify( this.startePos )
let oldpos = AFRAME.utils.coordinates.stringify( this.el.getAttribute('position') )
AFRAME.scenes[0].setAttribute("line__pull", `start: ${oldpos}; end : ${pos};`)
let d = this.startePos.distanceTo( this.el.getAttribute('position') )
// could show a preview state before release, e.g
let decimeters = Math.round(d*10)
console.log('pulling '+decimeters+' pages')
// update visible value instead, ideally under line but still facing user
let textPos = new THREE.Vector3()
textPos.lerpVectors(this.startePos, this.el.getAttribute('position'), .7)
this.decimtersEl.setAttribute('position', textPos )
this.decimtersEl.setAttribute('rotation', this.el.getAttribute('rotation') )
this.decimtersEl.setAttribute('value', decimeters )
},
released: function (evt) {
let d = this.startePos.distanceTo( this.el.getAttribute('position') )
console.log('This entity was released '+ d + 'm away from picked pos')
this.el.setAttribute('position', AFRAME.utils.coordinates.stringify( this.startePos ))
this.el.setAttribute('rotation', AFRAME.utils.coordinates.stringify( this.starteRot ))
AFRAME.scenes[0].removeAttribute("line__pull")
this.decimtersEl.remove()
},
},
});
// ==================================== utils on entities and classes ======================================================
function toggleVisibilityEntitiesFromClass(classname){
let entities = Array.from( document.querySelectorAll("."+classname) )
if (entities.length == 0) return
let state = entities[0].getAttribute("visible") // assume they are all the same
if (state)
entities.map( e => e.setAttribute("visible", "false"))
else
entities.map( e => e.setAttribute("visible", "true"))
}
function pushLeftClass(classname, value=.1){
Array.from( document.querySelectorAll("."+classname) ).map( e => e.object3D.position.x -= value)
}
function pushRightClass(classname, value=.1){
Array.from( document.querySelectorAll("."+classname) ).map( e => e.object3D.position.x += value)
}
function pushUpClass(classname, value=.1){
Array.from( document.querySelectorAll("."+classname) ).map( e => e.object3D.position.y += value)
}
function pushDownClass(classname, value=.1){
// can be used for accessibiliy, either directly or sampling e.g 10s after entering VR to lower based on the estimated user height
Array.from( document.querySelectorAll("."+classname) ).map( e => e.object3D.position.y -= value)
}
function pushBackClass(classname, value=.1){
Array.from( document.querySelectorAll("."+classname) ).map( e => e.object3D.position.z -= value)
}
function pushFrontClass(classname, value=.1){
Array.from( document.querySelectorAll("."+classname) ).map( e => e.object3D.position.z += value)
}
function toggleVisibilityAllGenerators(){
generators.split(" ").map( g => toggleVisibilityEntitiesFromClass(g) )
// not hidableassets though
}
function toggleVisibilityAll(){
toggleVisibilityAllGenerators()
toggleVisibilityEntitiesFromClass("hidableassets")
}
function toggleVisibilityAllButClass(classname){
generators.split(" ").filter( e => e != classname).map( g => toggleVisibilityEntitiesFromClass(g) )
toggleVisibilityEntitiesFromClass("hidableassets")
}
function switchSide(){
// mostly works... but event listeners are not properly removed. Quickly creates a mess, low performance and unpredictable.
document.querySelector("#"+sides[primarySide]+"Hand").removeAttribute("pinchprimary")
document.querySelector("#"+sides[secondarySide]+"Hand").removeAttribute("pinchsecondary")
document.querySelector("#"+sides[secondarySide]+"Hand").removeAttribute("wristattachsecondary")
document.querySelector("#"+sides[secondarySide]+"Hand").setAttribute("pinchprimary", "")
document.querySelector("#"+sides[primarySide]+"Hand").setAttribute("pinchsecondary", "")
document.querySelector("#"+sides[primarySide]+"Hand").setAttribute("wristattachsecondary", "target: #box")
if (primarySide == 0) {
secondarySide = 0
primarySide = 1
} else {
primarySide = 0
secondarySide = 1
}
}
function getIdFromPick(){
let id = null
let pp = selectedElements.filter( e => e.primary )
if (pp && pp[pp.length-1] && pp[pp.length-1].element ){
if (!pp[pp.length-1].element.id) pp[pp.length-1].element.id= "missingid_"+Date.now()
id = pp[pp.length-1].element.id
setFeedbackHUD(id)
}
return id
}
function getClassFromPick(){ // should be classes, for now assuming one
let classFound = null
let pp = selectedElements.filter( e => e.primary )
if (pp && pp[pp.length-1] && pp[pp.length-1].element ){
//if (!pp[pp.length-1].element.className) pp[pp.length-1].element.className= "missingclass"
// arguable
classFound = pp[pp.length-1].element.className
setFeedbackHUD(classFound)
}
return classFound
}
function getArrayFromClass(classname){
return Array.from( document.querySelectorAll("."+classname) )
}
function applyToClass(classname, callback, value){
// example applyToClass("template_object", (e, val ) => e.setAttribute("scale", val), ".1 .1 .2")
getArrayFromClass(classname).map( e => callback(e, value))
// could instead become a jxr shortcut, namely apply a set attribute to a class of entities
}
function addDropZone(position="0 1.4 -0.6", callback=setFeedbackHUD, radius=0.11){
// consider how this behavior could be similar to the wrist watch shortcut
// namely binding it to a jxr function
let el = document.createElement("a-sphere")
el.setAttribute("wireframe", true)
el.setAttribute("radius", radius)
el.setAttribute("position", position)
el.id = "dropzone_"+Date.now()
AFRAME.scenes[0].appendChild( el )
let sphere = new THREE.Sphere( AFRAME.utils.coordinates.parse( position ), radius )
// could become movable but would then need to move the matching sphere too
// could be a child of that entity
let pincher = document.querySelector('[pinchprimary]')
pincher.addEventListener('pinchended', function (event) {
if (selectedElements.length){
let lastDrop = selectedElements[selectedElements.length-1]
if ((Date.now() - lastDrop.timestamp) < 1000){
if (sphere.containsPoint( lastDrop.element.getAttribute("position"))){
// should be a threejs sphere proper, not a mesh
console.log("called back" )
callback( lastDrop.selectedElement )
}
}
}
})
// never unregister
return el
}
// ==================================== facilitating debugging ======================================================
function makeAnchorsVisibleOnTargets(){
targets.map( t => {
let controlSphere = document.createElement("a-sphere")
controlSphere.setAttribute("radius", 0.05)
controlSphere.setAttribute("color", "blue")
controlSphere.setAttribute("wireframe", "true")
controlSphere.setAttribute("segments-width", 8)
controlSphere.setAttribute("segments-height", 8)
t.appendChild( controlSphere )
}) // could provide a proxy to be able to monitor efficiently
}
function switchToWireframe(){
let model = document.querySelector("#environment")?.object3D
if (model) model.traverse( o => { if (o.material) {
let visible = !o.material.wireframe
o.material.wireframe = visible;
o.material.opacity = visible ? 0.05 : 1;
o.material.transparent = visible;
} })
}
// avoiding setOnDropFromAttribute() as it is not idiosyncratic and creates timing issues

@ -0,0 +1,8 @@
{
"dependencies": {
"@elasticemail/elasticemail-client": "^4.0.23",
"express": "^4.21.1",
"ip": "^2.0.1",
"node-html-to-image": "^5.0.0"
}
}

@ -0,0 +1,14 @@
#!/bin/bash
cd public
echo -n '[' > ../pack.json; for X in *;
do
echo -n "{\"filename\":\"$X\", \"content\":\"$(base64 -w0 $X)\"" >> ../pack.json;
# curl --insecure -I https://192.168.0.129:3000/$X | grep "Content-Type" | sed 's/;.*//' | sed 's/\(.*\): \(.*\)/,"\1":"\2"/' #>> ../pack.json;
echo -n ",\"contenttype\":\"" >> ../pack.json;
curl --insecure -I -s -o /dev/null -w '%header{Content-Type}' https://192.168.0.129:3000/$X | sed 's/;.*//' >> ../pack.json;
echo -n "\"}," >> ../pack.json;
done;
sed -i "s/.$//" ../pack.json;
echo -n "]" >> ../pack.json

@ -0,0 +1,194 @@
-- Pandoc reader for PMWiki format: https://www.pmwiki.org/wiki/PmWiki/MarkupMasterIndex
-- Using LPeg: https://www.inf.puc-rio.br/~roberto/lpeg/
-- Inspired by https://pandoc.org/custom-readers.html
local P, S, R, Cf, Cc, Ct, V, Cs, Cg, Cb, B, C, Cmt =
lpeg.P, lpeg.S, lpeg.R, lpeg.Cf, lpeg.Cc, lpeg.Ct, lpeg.V,
lpeg.Cs, lpeg.Cg, lpeg.Cb, lpeg.B, lpeg.C, lpeg.Cmt
local whitespacechar = S(" \t\r\n")
local specialchar = S("/*~[]\\{}|")
local wordchar = (1 - (whitespacechar + specialchar))
local spacechar = S(" \t")
local newline = P"\r"^-1 * P"\n"
local blankline = spacechar^0 * newline
local endline = newline * #-blankline
local endequals = spacechar^0 * P"="^0 * spacechar^0 * newline
local cellsep = spacechar^0 * P"|"
local apostrophe = string.char(39)
local doubleApo = P(apostrophe) * P(apostrophe)
local fenced = '```\n%s\n```\n'
local cellsep = spacechar^0 * P"||"
local function trim(s)
return (s:gsub("^%s*(.-)%s*$", "%1"))
end
local function ListItem(lev, ch)
local start
if ch == nil then
start = S"*#"
else
start = P(ch)
end
local subitem = function(c)
if lev < 6 then
return ListItem(lev + 1, c)
else
return (1 - 1) -- fails
end
end
local parser = spacechar^0
* start^lev
* #(- start)
* spacechar^0
* Ct((V"Inline" - (newline * spacechar^0 * S"*#"))^0)
* newline
* (Ct(subitem("*")^1) / pandoc.BulletList
+
Ct(subitem("#")^1) / pandoc.OrderedList
+
Cc(nil))
/ function (ils, sublist)
return { pandoc.Plain(ils), sublist }
end
return parser
end
-- Grammar
G = P{ "Doc",
Doc = Ct(V"Block"^0)
/ pandoc.Pandoc ;
Block = blankline^0
* ( V"IndentedBlock"
+ V"Header"
+ V"HorizontalRule"
+ V"CodeBlock"
+ V"List"
+ V"Table"
+ V"Para"
) ;
IndentedBlock = C((spacechar^1
* (1 - newline)^1
* newline)^1
)
/ function(text)
block = pandoc.RawBlock('markdown', fenced:format(text))
return block
end;
CodeBlock = P"[@"
* blankline
* C((1 - (newline * P"@]"))^0)
* newline
* P"@]"
/ function(text)
block = pandoc.RawBlock('markdown', fenced:format(text))
return block
end;
List = V"BulletList"
+ V"OrderedList" ;
BulletList = Ct(ListItem(1,'*')^1)
/ pandoc.BulletList ;
OrderedList = Ct(ListItem(1,'#')^1)
/ pandoc.OrderedList ;
Table = V"TableProperties"
* (V"TableHeader" + Cc{})
* Ct(V"TableRow"^1)
/ function(headrow, bodyrows)
local numcolumns = #(bodyrows[1])
local aligns = {}
local widths = {}
for i = 1,numcolumns do
aligns[i] = pandoc.AlignDefault
widths[i] = 0
end
return pandoc.utils.from_simple_table(
pandoc.SimpleTable({}, aligns, widths, headrow, bodyrows))
end ;
TableProperties = cellsep
* spacechar^0
* P("border=")
* (1 - newline)^1
* newline;
TableHeader = Ct(V"HeaderCell"^1)
* cellsep^-1
* spacechar^0
* newline ;
TableRow = Ct(V"BodyCell"^1)
* cellsep^-1
* spacechar^0
* newline ;
HeaderCell = cellsep
* P"!"^-1
* spacechar^0
* Ct((V"Inline" - (newline + cellsep))^0)
/ function(ils) return { pandoc.Plain(ils) } end ;
BodyCell = cellsep
* spacechar^0
* Ct((V"Inline" - (newline + cellsep))^0)
/ function(ils) return { pandoc.Plain(ils) } end ;
Para = Ct(V"Inline"^1)
* newline
/ pandoc.Para ;
HorizontalRule = spacechar^0
* P"----"
* spacechar^0
* newline
/ pandoc.HorizontalRule;
Header = (P("!")^1 / string.len)
* spacechar^0
* Ct((V"Inline" - endequals)^1)
* endequals
/ pandoc.Header;
Inline = V"Link"
+ V"Url"
+ V"Code"
+ V"Bold"
+ V"Emph"
+ V"Strikeout"
+ V"Str"
+ V"Space"
+ V"Special";
Link = P"[["
* C((1 - (P"]]" + P"|"))^0)
* (P"|" * Ct((V"Inline" - P"]]")^1))^-1
* P"]]"
/ function(url, desc)
local txt = desc or {pandoc.Str(url)}
return pandoc.Link(txt, url)
end;
Url = C(
P"http"
* P"s"^-1
* P"://"
* (1 - whitespacechar)^1
)
/ function(url)
return pandoc.Link(url, url)
end;
Code = P'@@'
* C((1 - P'@@')^0)
* P'@@'
/ trim / pandoc.Code;
Emph = P"''"
* C(((wordchar + whitespacechar) - P"''")^1)
* P"''"
/ pandoc.Emph;
Bold = P"'''"
* C(((wordchar + whitespacechar) - P"'''")^1)
* P"'''"
/ pandoc.Strong;
Strikeout = P"{-"
* C(((wordchar + whitespacechar) - P"-}")^1)
* P"-}"
/ pandoc.Strikeout;
Str = wordchar^1
/ pandoc.Str;
Special = specialchar
/ pandoc.Str;
Space = spacechar^1
/ pandoc.Space ;
}
function Reader(input, reader_options)
return lpeg.match(G, tostring(input))
end

@ -0,0 +1,26 @@
{
"default" : [
{"selector":"#start_file_sloan_testtxt_end_file_hello_worldtxt", "attribute":"line", "value": "color:blue"},
{"selector":"a-sky", "attribute":"color", "value": "lightblue"},
{"selector":".notes", "attribute":"color", "value": "purple"},
{"selector":".notes", "attribute":"outline-color", "value": "darkblue"},
{"selector":"a-troika-text a-plane", "attribute":"color", "value": "white"},
{"selector":"a-troika-text a-triangle", "attribute":"color", "value": "gray"}
],
"light" : [
{"selector":"#start_file_sloan_testtxt_end_file_hello_worldtxt", "attribute":"line", "value": "color:blue"},
{"selector":"a-sky", "attribute":"color", "value": "gray"},
{"selector":".notes", "attribute":"color", "value": "black"},
{"selector":".notes", "attribute":"outline-color", "value": "white"},
{"selector":"a-troika-text a-plane", "attribute":"color", "value": "red"},
{"selector":"a-troika-text a-triangle", "attribute":"color", "value": "darkred"}
],
"print" : [
{"selector":"#start_file_sloan_testtxt_end_file_hello_worldtxt", "attribute":"line", "value": "color:brown"},
{"selector":"a-sky", "attribute":"color", "value": "#EEE"},
{"selector":".notes", "attribute":"color", "value": "black"},
{"selector":".notes", "attribute":"outline-color", "value": "white"},
{"selector":"a-troika-text a-plane", "attribute":"color", "value": "lightyellow"},
{"selector":"a-troika-text a-triangle", "attribute":"color", "value": "orange"}
]
}
Loading…
Cancel
Save