Skip to main content

Posts about Node-Red

Node-Red Website Alerts

Node-RED is a flow-based programming tool, originally developed by IBM’s Emerging Technology Services team and now a part of the JS Foundation. A recent question on /r/selfhosted about what selfhosted service you are missing sparked my interest. A user by the name forthedatahorde mentioned they wanted the ability to monitor arbitrary websites. I figured Node-Red is just the tool to comfortably fill this gap.

I made two options. One using readability.js for a bit of a generic solution and one using css selectors for more targeted watching needs.

The Readability.js Version:

  1. Fetches the site with a HTTP get request.

  2. Runs it through readability.js

  3. Hashes the text

  4. Compares it with an old hash

  5. Emails the change

Required the packages node-red-contrib-md5 and node-red-contrib-readability

image03

[{"id":"c8beb17e.4513c8","type":"http request","z":"1a1be165.968ed7","name":"","method":"GET","ret":"txt","paytoqs":false,"url":"","tls":"","proxy":"","authType":"","x":310,"y":100,"wires":[["8a94f0a4.a1f488"]]},{"id":"ae660213.061188","type":"readability","z":"1a1be165.968ed7","name":"","x":310,"y":180,"wires":[["597a982a.8c9bb"]]},{"id":"8a94f0a4.a1f488","type":"switch","z":"1a1be165.968ed7","name":"","property":"statusCode","propertyType":"msg","rules":[{"t":"eq","v":"200","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":290,"y":140,"wires":[["ae660213.061188"]]},{"id":"c361fc4a.44f6b8","type":"change","z":"1a1be165.968ed7","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"payload.content","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":500,"y":300,"wires":[["4e96af67.1febe"]]},{"id":"597a982a.8c9bb","type":"md5","z":"1a1be165.968ed7","name":"","fieldToHash":"payload.text","fieldTypeToHash":"msg","hashField":"md5","hashFieldType":"msg","x":470,"y":140,"wires":[["8e8bc713.5693f"]]},{"id":"ed8cd02.da0d03","type":"switch","z":"1a1be165.968ed7","name":"","property":"md5","propertyType":"msg","rules":[{"t":"neq","v":"old_hash","vt":"msg"}],"checkall":"true","repair":false,"outputs":1,"x":610,"y":180,"wires":[["d52d318b.d1ac58"]]},{"id":"5cfa0e17.f15bf","type":"change","z":"1a1be165.968ed7","name":"","rules":[{"t":"set","p":"url","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":310,"y":60,"wires":[["c8beb17e.4513c8"]]},{"id":"4ab25748.add748","type":"change","z":"1a1be165.968ed7","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"url","tot":"msg"},{"t":"change","p":"topic","pt":"msg","from":"^(.*)$","fromt":"re","to":"Update to $1 detected","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":500,"y":260,"wires":[["c361fc4a.44f6b8"]]},{"id":"4e96af67.1febe","type":"e-mail","z":"1a1be165.968ed7","server":"smtp.gmail.com","port":"465","secure":true,"tls":true,"name":"","dname":"","x":690,"y":300,"wires":[]},{"id":"b7b6adc.1b5c3d","type":"inject","z":"1a1be165.968ed7","name":"rammstein.de","topic":"","payload":"https://www.rammstein.de/de/","payloadType":"str","repeat":"21600","crontab":"","once":false,"onceDelay":0.1,"x":120,"y":60,"wires":[["5cfa0e17.f15bf"]]},{"id":"8e8bc713.5693f","type":"function","z":"1a1be165.968ed7","name":"Get Hash","func":"try{\n    msg.old_hash = flow.get(msg.url);\n} catch(e) {\n    msg.old_hash = \"0\";\n}\nreturn msg;","outputs":1,"noerr":0,"x":620,"y":140,"wires":[["ed8cd02.da0d03"]]},{"id":"d52d318b.d1ac58","type":"function","z":"1a1be165.968ed7","name":"Set Hash","func":"flow.set(msg.url, msg.md5);\nreturn msg;","outputs":1,"noerr":0,"x":480,"y":220,"wires":[["4ab25748.add748"]]}]

The CSS Selector Version:

  1. Fetches the site with a HTTP get request.

  2. Filters the resulting HTML with a css selector

  3. Hashes the html

  4. Compares it with an old hash

  5. Emails the change

Required the packages node-red-contrib-md5

image02

[{"id":"c8beb17e.4513c8","type":"http request","z":"1a1be165.968ed7","name":"","method":"GET","ret":"txt","paytoqs":false,"url":"","tls":"","proxy":"","authType":"","x":350,"y":160,"wires":[["8a94f0a4.a1f488"]]},{"id":"8a94f0a4.a1f488","type":"switch","z":"1a1be165.968ed7","name":"","property":"statusCode","propertyType":"msg","rules":[{"t":"eq","v":"200","vt":"str"}],"checkall":"true","repair":false,"outputs":1,"x":330,"y":200,"wires":[["28487abf.9cf7be"]]},{"id":"597a982a.8c9bb","type":"md5","z":"1a1be165.968ed7","name":"","fieldToHash":"payload","fieldTypeToHash":"msg","hashField":"md5","hashFieldType":"msg","x":330,"y":280,"wires":[["8e8bc713.5693f"]]},{"id":"ed8cd02.da0d03","type":"switch","z":"1a1be165.968ed7","name":"","property":"md5","propertyType":"msg","rules":[{"t":"neq","v":"old_hash","vt":"msg"}],"checkall":"true","repair":false,"outputs":1,"x":330,"y":360,"wires":[["d52d318b.d1ac58"]]},{"id":"5cfa0e17.f15bf","type":"change","z":"1a1be165.968ed7","name":"","rules":[{"t":"set","p":"url","pt":"msg","to":"payload","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":350,"y":120,"wires":[["c8beb17e.4513c8"]]},{"id":"4ab25748.add748","type":"change","z":"1a1be165.968ed7","name":"","rules":[{"t":"set","p":"topic","pt":"msg","to":"url","tot":"msg"},{"t":"change","p":"topic","pt":"msg","from":"^(.*)$","fromt":"re","to":"Update to $1 detected","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":520,"y":400,"wires":[["9a394414.9956a"]]},{"id":"b7b6adc.1b5c3d","type":"inject","z":"1a1be165.968ed7","name":"rammstein.de","topic":"","payload":"https://www.rammstein.de/de/","payloadType":"str","repeat":"","crontab":"","once":false,"onceDelay":0.1,"x":110,"y":80,"wires":[["6500331b.56333c"]]},{"id":"8e8bc713.5693f","type":"function","z":"1a1be165.968ed7","name":"Get Hash","func":"try{\n    msg.old_hash = flow.get(msg.url);\n} catch(e) {\n    msg.old_hash = \"0\";\n}\nreturn msg;","outputs":1,"noerr":0,"x":340,"y":320,"wires":[["ed8cd02.da0d03"]]},{"id":"d52d318b.d1ac58","type":"function","z":"1a1be165.968ed7","name":"Set Hash","func":"flow.set(msg.url, msg.md5);\nreturn msg;","outputs":1,"noerr":0,"x":500,"y":360,"wires":[["4ab25748.add748"]]},{"id":"28487abf.9cf7be","type":"html","z":"1a1be165.968ed7","name":"","property":"payload","outproperty":"payload","tag":"","ret":"html","as":"single","x":330,"y":240,"wires":[["b8739927.661e18"]]},{"id":"6500331b.56333c","type":"change","z":"1a1be165.968ed7","name":"Selector css","rules":[{"t":"set","p":"select","pt":"msg","to":".resume-view-news","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":110,"y":120,"wires":[["5cfa0e17.f15bf"]]},{"id":"b8739927.661e18","type":"change","z":"1a1be165.968ed7","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"payload[0]","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":520,"y":240,"wires":[["597a982a.8c9bb"]]},{"id":"9a394414.9956a","type":"e-mail","z":"1a1be165.968ed7","server":"smtp.gmail.com","port":"465","secure":true,"tls":true,"name":"","dname":"","x":710,"y":400,"wires":[]}]

I hope someone finds this helpful/interesting. Let me know if you have a better solution in the comments down below.


Fixing a Patreon feed's cover artwork

Antennapod With the recent Pocketcast PR blunder I finally decided to jump back to open source Antennapod. This has been painless especially with the introduction of the Remove silence feature, the main feature that kept me with Pocketcasts for so long.

The only issue I had was a Private feed with broken cover artwork, this was frustrating but it looks to be an issue on Patreons end. Thankfully Node Red is available to rescue the situation!

The flow is really simple, the only additional node I have added is the node-red-contrib-httpauth.

  1. On a HTTP request, fetch the RSS feed.
  2. Convert from XML to and Object.
  3. Replace msg.payload.rss.channel[0].image[0].url[0] with a good url from the podcasters website.
  4. Create the txt/xml headers
  5. Return the fixed RSS Feed

Node-Red Flow

[{"id":"8d2a4ad1.4599a","type":"http request","z":"d437ad18.0999c","name":"","method":"GET","ret":"txt","paytoqs":false,"url":"https://www.patreon.com/rss/yaddayada","tls":"","proxy":"","authType":"","x":367.5,"y":31,"wires":[["4c5cdedd.6772c8"]]},{"id":"4c5cdedd.6772c8","type":"xml","z":"d437ad18.0999c","name":"XML To Object","property":"payload","attr":"","chr":"","x":230.5,"y":78,"wires":[["adbd8c2e.7fbcd"]]},{"id":"7475a34e.9f953c","type":"http in","z":"d437ad18.0999c","name":"rss","url":"/mystupidRSS","method":"get","upload":false,"swaggerDoc":"","x":69.5,"y":31,"wires":[["26c3aede.f41d4a"]]},{"id":"26c3aede.f41d4a","type":"node-red-contrib-httpauth","z":"d437ad18.0999c","name":"","file":"","cred":"","authType":"Basic","realm":"","username":"","password":"","hashed":false,"x":213.5,"y":31,"wires":[["8d2a4ad1.4599a"]]},{"id":"56731f49.b7f77","type":"xml","z":"d437ad18.0999c","name":"Object to XML","property":"payload","attr":"","chr":"","x":647.5,"y":79,"wires":[["5aa22446.612044"]]},{"id":"5aa22446.612044","type":"change","z":"d437ad18.0999c","name":"Set Headers","rules":[{"t":"set","p":"headers","pt":"msg","to":"{}","tot":"json"},{"t":"set","p":"headers.content-type","pt":"msg","to":"text/xml","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":219,"y":131,"wires":[["8df07127.e9d0c8"]]},{"id":"8df07127.e9d0c8","type":"http response","z":"d437ad18.0999c","name":"","statusCode":"","headers":{},"x":391,"y":131,"wires":[]},{"id":"adbd8c2e.7fbcd","type":"change","z":"d437ad18.0999c","name":"Replace Cover Image","rules":[{"t":"set","p":"payload.rss.channel[0].image[0].url[0]","pt":"msg","to":"https://example.com/cover_art.png","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":440.5,"y":79,"wires":[["56731f49.b7f77"]]}]