{"id":15706733,"url":"https://github.com/jexp/mastodon-import-neo4j","last_synced_at":"2025-04-30T12:44:31.906Z","repository":{"id":66743961,"uuid":"564949862","full_name":"jexp/mastodon-import-neo4j","owner":"jexp","description":"Importing Mastodon Toots into Neo4j","archived":false,"fork":false,"pushed_at":"2022-11-11T23:23:43.000Z","size":3811,"stargazers_count":5,"open_issues_count":1,"forks_count":0,"subscribers_count":2,"default_branch":"main","last_synced_at":"2025-03-30T16:13:54.645Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":null,"has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/jexp.png","metadata":{"files":{"readme":"README.adoc","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2022-11-11T22:26:26.000Z","updated_at":"2022-11-30T14:38:23.000Z","dependencies_parsed_at":null,"dependency_job_id":"e3f956d5-5064-4782-99e5-e25f1e09f8fc","html_url":"https://github.com/jexp/mastodon-import-neo4j","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jexp%2Fmastodon-import-neo4j","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jexp%2Fmastodon-import-neo4j/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jexp%2Fmastodon-import-neo4j/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/jexp%2Fmastodon-import-neo4j/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/jexp","download_url":"https://codeload.github.com/jexp/mastodon-import-neo4j/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":251703084,"owners_count":21630158,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-10-03T20:27:27.176Z","updated_at":"2025-04-30T12:44:31.881Z","avatar_url":"https://github.com/jexp.png","language":null,"funding_links":[],"categories":[],"sub_categories":[],"readme":"== Importing Mastodon Toots into Neo4j\n\n=== Mastodon REST API endpoints:\n\n* public endpoints https://docs.joinmastodon.org/client/public/\n// https://docs.joinmastodon.org/api/rate-limits/\n\n// call apoc.load.json(\"\")\n\n* server peers: https://mastodon.social//api/v1/instance/peers\n* users: https://mastodon.social//api/v1/directory\n* public timeline: https://mastodon.social/api/v1/timelines/public\n* tags: https://mastodon.social/api/v1/timelines/tag/tagname\n\n.Example Response\n[source,json]\n----\n[\n  {\n    \"id\": \"109327593515660598\",\n    \"created_at\": \"2022-11-11T22:44:48.185Z\",\n    \"in_reply_to_id\": null,\n    \"in_reply_to_account_id\": null,\n    \"sensitive\": false,\n    \"spoiler_text\": \"\",\n    \"visibility\": \"public\",\n    \"language\": \"en\",\n    \"uri\": \"https://mastodon.social/users/eschaton/statuses/109327593515660598\",\n    \"url\": \"https://mastodon.social/@eschaton/109327593515660598\",\n    \"replies_count\": 0,\n    \"reblogs_count\": 0,\n    \"favourites_count\": 0,\n    \"edited_at\": null,\n    \"content\": \"\u003cp\u003eI hope Elon Musk is fired by Tesla’s board and the original founders are brought back Steve Jobs-style.\u003c/p\u003e\",\n    \"reblog\": null,\n    \"application\": {\n      \"name\": \"Mastodon for iOS\",\n      \"website\": \"https://app.joinmastodon.org/ios\"\n    },\n    \"account\": {\n      \"id\": \"131321\",\n      \"username\": \"eschaton\",\n      \"acct\": \"eschaton\",\n      \"display_name\": \"Chris Hanson\",\n      \"locked\": false,\n      \"bot\": false,\n      \"discoverable\": true,\n      \"group\": false,\n      \"created_at\": \"2017-05-08T00:00:00.000Z\",\n      \"note\": \"\u003cp\u003eWork on tools for people to do work. Play with old computers in my spare time. Only ever speaking for myself.\u003c/p\u003e\",\n      \"url\": \"https://mastodon.social/@eschaton\",\n      \"avatar\": \"https://files.mastodon.social/accounts/avatars/000/131/321/original/26ff9bdc28975579.jpg\",\n      \"avatar_static\": \"https://files.mastodon.social/accounts/avatars/000/131/321/original/26ff9bdc28975579.jpg\",\n      \"header\": \"https://files.mastodon.social/accounts/headers/000/131/321/original/4b4dd27e825ca163.png\",\n      \"header_static\": \"https://files.mastodon.social/accounts/headers/000/131/321/original/4b4dd27e825ca163.png\",\n      \"followers_count\": 299,\n      \"following_count\": 215,\n      \"statuses_count\": 90,\n      \"last_status_at\": \"2022-11-11\",\n      \"noindex\": false,\n      \"emojis\": [],\n      \"fields\": [\n        {\n          \"name\": \"Blog\",\n          \"value\": \"\u003ca href=\\\"http://eschatologist.net/blog/\\\" target=\\\"_blank\\\" rel=\\\"nofollow noopener noreferrer me\\\"\u003e\u003cspan class=\\\"invisible\\\"\u003ehttp://\u003c/span\u003e\u003cspan class=\\\"\\\"\u003eeschatologist.net/blog/\u003c/span\u003e\u003cspan class=\\\"invisible\\\"\u003e\u003c/span\u003e\u003c/a\u003e\",\n          \"verified_at\": null\n        }\n      ]\n    },\n    \"media_attachments\": [],\n    \"mentions\": [],\n    \"tags\": [],\n    \"emojis\": [],\n    \"card\": null,\n    \"poll\": null\n  }\n]\n----\n\n=== Data Model\n\n.Simple Data Model\nimage::mastodon-model.png[]\n\n=== Load Scripts\n\n.Create Constraints\n[source,cypher]\n----\ncreate constraint message_id if not exists FOR (m:Message) REQUIRE (m.id) is unique;\ncreate constraint user_id if not exists FOR (u:User) REQUIRE (u.acct) is unique;\ncreate constraint tag_name if not exists FOR (t:Tag) REQUIRE (t.name) is unique;\n----\n\nTODO proper pagination?\n\n.Load Public Timeline\n[source,cypher]\n----\n:auto unwind range(0,4000,40) as skip\ncall { with skip\ncall apoc.load.json(\"https://mastodon.social/api/v1/timelines/public?limit=40\u0026skip=\"+skip) yield value\nunwind value as v\nWITH v, v.account as a\nMERGE (m:Message {id:v.id})\nON CREATE SET m += v {.sensitive, .language, .uri, .replies_count, .favourites_count, .content },\nm.created_at = datetime(v.created_at)\n\nMERGE (u:User {acct:a.acct})\nON CREATE SET u += a {.id, .locked, .bot, .discoverable, .group, .note, .url, .avatar, \n.followers_count, .following_count, .statuses_count }, \nu.created_at = datetime(a.created_at),\nu.last_status_at = date(a.last_status_at),\nu.twitter = [f IN a.fields WHERE f.name = 'Twitter' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.pronouns = [f IN a.fields WHERE f.name = 'Pronouns' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.website = [f IN a.fields WHERE f.name IN ['Website','Homepage','Site','Web'] | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.github = [f IN a.fields WHERE f.name = 'Github' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.reddit = [f IN a.fields WHERE f.name = 'Reddit' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.location = [f IN a.fields WHERE f.name = 'Location' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.lang = [f IN a.fields WHERE f.name = 'Lang' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.support = [f IN a.fields WHERE f.name = 'Support' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0]\n\nMERGE (u)-[:POSTED]-\u003e(m)\nWITH * \ncall { with v,m\n    UNWIND v.tags as t\n    MERGE (tag:Tag {name:t.name}) // ON CREATE SET tag.url = t.url\n    MERGE (m)-[:TAGGED]-\u003e(tag)\n    RETURN count(*) as tags\n}\n\nWITH * WHERE v.in_reply_to_id is not null\nMERGE (reply:Message {id:v.in_reply_to_id})\nMERGE (m)-[:REPLIED_TO]-\u003e(reply)\n} in transactions of 1 row;\n----\n\n.Load Tags\n[source,cypher]\n----\n:auto MATCH (t:Tag)\nWITH t.name as name\ncall { with name\ncall apoc.load.json(\"https://mastodon.social/api/v1/timelines/tag/+\"+name) yield value\nunwind value as v\nWITH v, v.account as a\nMERGE (m:Message {id:v.id})\nON CREATE SET m += v {.sensitive, .language, .uri, .replies_count, .favourites_count, .content },\nm.created_at = datetime(v.created_at)\n\nMERGE (u:User {acct:a.acct})\nON CREATE SET u += a {.id, .locked, .bot, .discoverable, .group, .note, .url, .avatar, \n.followers_count, .following_count, .statuses_count }, \nu.created_at = datetime(a.created_at),\nu.last_status_at = date(a.last_status_at),\nu.twitter = [f IN a.fields WHERE f.name = 'Twitter' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.pronouns = [f IN a.fields WHERE f.name = 'Pronouns' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.website = [f IN a.fields WHERE f.name IN ['Website','Homepage','Site','Web'] | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.github = [f IN a.fields WHERE f.name = 'Github' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.reddit = [f IN a.fields WHERE f.name = 'Reddit' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.location = [f IN a.fields WHERE f.name = 'Location' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.lang = [f IN a.fields WHERE f.name = 'Lang' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.support = [f IN a.fields WHERE f.name = 'Support' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0]\n\nMERGE (u)-[:POSTED]-\u003e(m)\nWITH * \ncall { with v,m\n    UNWIND v.tags as t\n    MERGE (tag:Tag {name:t.name}) // ON CREATE SET tag.url = t.url\n    MERGE (m)-[:TAGGED]-\u003e(tag)\n    RETURN count(*) as tags\n}\n\nWITH * WHERE v.in_reply_to_id is not null\nMERGE (reply:Message {id:v.in_reply_to_id})\nMERGE (m)-[:REPLIED_TO]-\u003e(reply)\n} in transactions of 1 row;\n----\n\n.Load User\n[source,cypher]\n----\n:auto unwind range(0,4000,40) as skip\ncall { with skip\ncall apoc.load.json(\"https://mastodon.social//api/v1/directory?limit=40\u0026skip=\"+skip) yield value as a\n\nMERGE (u:User {acct:a.acct})\nON CREATE SET u += a {.id, .locked, .bot, .discoverable, .group, .note, .url, .avatar, \n.followers_count, .following_count, .statuses_count }, \nu.created_at = datetime(a.created_at),\nu.last_status_at = date(a.last_status_at),\nu.twitter = [f IN a.fields WHERE f.name = 'Twitter' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.pronouns = [f IN a.fields WHERE f.name = 'Pronouns' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.website = [f IN a.fields WHERE f.name IN ['Website','Homepage','Site','Web'] | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.github = [f IN a.fields WHERE f.name = 'Github' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.reddit = [f IN a.fields WHERE f.name = 'Reddit' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.location = [f IN a.fields WHERE f.name = 'Location' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.lang = [f IN a.fields WHERE f.name = 'Lang' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0],\nu.support = [f IN a.fields WHERE f.name = 'Support' | apoc.text.regexGroups(f.value, 'href=\"([^\"]+)\"')[0][1]][0]\n} in transactions of 1 row;\n----\n\n=== Example Graph\n\n.Full Graph\nimage::mastodon.svg[]\n\n.Readable subset\nimage::mastodon-subset.png[]\n\n\n.Larger Graph in Bloom\nimage::mastodon-bloom-graph.png[]","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fjexp%2Fmastodon-import-neo4j","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fjexp%2Fmastodon-import-neo4j","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fjexp%2Fmastodon-import-neo4j/lists"}