Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit bcd4a54

Browse files
kickscondorkickscondor
committedMay 29, 2021
Improve handling of non-string properties, better TikTok support.
1 parent 1062366 commit bcd4a54

File tree

3 files changed

+73
-72
lines changed

3 files changed

+73
-72
lines changed
 

‎defs/social.json

Lines changed: 71 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
{
22
"domains": {
33
"api.twitch.tv": {"headers:Client-Id": "vnvcshh0aehfkq0bvu6rzpbtaf25et",
4-
"headers:Authorization": "Bearer mjal5b1xpg5k47awdcx2qv9f9v0sut"},
4+
"headers:Authorization": "Bearer 6n8qc1brac67r1ox65m63lf9dtya6n"},
55
"api.twitter.com": {"headers:Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs=1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
66
"headers:DNT": "1", "headers:Content-Type": "application/json",
77
"headers:X-Twitter-Active-User": "yes", "headers:Authority": "api.twitter.com",
88
"headers:Accept-Language": "en-US,en;q=0.9", "headers:Accept": "*/*"},
9-
"api.tumblr.com": {"query:api_key": "fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4"},
9+
"api.tumblr.com": {"headers:Authorization": "Bearer aIcXSOoTtqrzR8L8YEIOmBeW94c3FmbSNSWAUbxsny9KKx5VFh"},
1010
"m.facebook.com": {"headers:User-Agent": "curl/7.68.0"},
1111
"www.youtube.com": {"headers:User-Agent": "curl/7.68.0"}
1212
},
@@ -138,22 +138,22 @@
138138
"jsonfeed": {
139139
"rules": {
140140
"entry": [
141-
{"var": "out:title", "op": "$.title"},
142-
{"var": "out:html", "op": "$.content_html"},
141+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
142+
{"var": "out:html", "op": "$.content_html", "mod": ["str"]},
143143
{"var": "out:url", "op": "$.url", "mod": ["url"]},
144144
{"var": "out:publishedAt", "op": "$.date_published", "mod": ["date"]},
145145
{"var": "out:updatedAt", "op": "$.date_modified", "mod": ["date"]}
146146
]
147147
},
148148
"acceptJson": [
149-
{"var": "out:title", "op": "$.title"},
149+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
150150
{"var": "out:photos:avatar", "op": "$.icon", "mod": ["url"]},
151-
{"var": "out:photos:favicon", "op": "$.favicon"},
152-
{"var": "out:description", "op": "$.description"},
153-
{"var": "out:url", "op": "$.home_page_url"},
151+
{"var": "out:photos:favicon", "op": "$.favicon", "mod": ["url"]},
152+
{"var": "out:description", "op": "$.description", "mod": ["str"]},
153+
{"var": "out:url", "op": "$.home_page_url", "mod": ["url"]},
154154
{"op": "$._status.*", "var": "out:status", "acceptJson": [
155-
{"var": "out:label", "op": "$.label"},
156-
{"var": "out:type", "op": "$.type"},
155+
{"var": "out:label", "op": "$.label", "mod": ["str"]},
156+
{"var": "out:type", "op": "$.type", "mod": ["str"]},
157157
{"rule": "entry"}
158158
]},
159159
{"op": "$.items.*", "var": "out:posts", "acceptJson": [
@@ -266,7 +266,7 @@
266266
{"var": "itemId", "op": "$.item_id"},
267267
{"var": "itemType", "op": "$.item_type"},
268268
{"var": "out:url", "op": ["=${items:$itemType-$itemId}", "=/releases"], "mod": ["url"]},
269-
{"var": "out:title", "op": "$.title"},
269+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
270270
{"var": "out:publishedAt", "op": "$.release_date", "mod": ["date"]}
271271
]}
272272
]
@@ -313,19 +313,19 @@
313313
{"op": "//script[contains(text(), 'profile_pic_url')]", "match": "({.+});",
314314
"acceptJson": [
315315
{"op": "$.entry_data.ProfilePage[0].graphql.user", "acceptJson": [
316-
{"var": "out:title", "op": "$.full_name"},
317-
{"var": "out:description", "op": "$.biography"},
318-
{"var": "out:photos:avatar", "op": "$.profile_pic_url"},
316+
{"var": "out:title", "op": "$.full_name", "mod": ["str"]},
317+
{"var": "out:description", "op": "$.biography", "mod": ["str"]},
318+
{"var": "out:photos:avatar", "op": "$.profile_pic_url", "mod": ["url"]},
319319
{"op": "$.edge_owner_to_timeline_media.edges.*", "var": "out:posts",
320320
"acceptJson": [
321321
{"var": "shortcode", "op": "$.node.shortcode"},
322322
{"var": "out:url", "op": "=https://www.instagram.com/p/$shortcode"},
323323
{"var": "out:publishedAt", "op": "$.node.taken_at_timestamp",
324324
"mod": ["int", "*1000", "date"]},
325-
{"var": "out:graphic:full", "op": "$.node.display_url"},
326-
{"var": "out:graphic:thumb", "op": "$.node.thumbnail_src"},
325+
{"var": "out:graphic:full", "op": "$.node.display_url", "mod": ["url"]},
326+
{"var": "out:graphic:thumb", "op": "$.node.thumbnail_src", "mod": ["url"]},
327327
{"var": "out:text", "op": ["$.node.edge_media_to_caption.edges[0].node.text",
328-
"$.node.accessibility_caption"]}
328+
"$.node.accessibility_caption"], "mod": ["str"]}
329329
]}
330330
]}
331331
]}
@@ -355,13 +355,13 @@
355355
{"var": "out:photos:avatar", "op": "//img[@class='avatar-image']/@src"},
356356
{"op": "//div[@data-projects]/@data-projects", "var": "out:posts",
357357
"match": ".", "acceptJson": [
358-
{"var": "out:title", "op": "$.name"},
359-
{"var": "out:url", "op": "$.urls.web.project"},
358+
{"var": "out:title", "op": "$.name", "mod": ["str"]},
359+
{"var": "out:url", "op": "$.urls.web.project", "mod": ["url"]},
360360
{"var": "out:publishedAt", "op": "$.launched_at",
361361
"mod": ["int", "*1000", "date"]},
362362
{"var": "out:updatedAt", "op": "$.state_changed_at",
363363
"mod": ["int", "*1000", "date"]},
364-
{"var": "out:text", "op": "$.blurb"}
364+
{"var": "out:text", "op": "$.blurb", "mod": ["str"]}
365365
]}
366366
]
367367
},
@@ -371,12 +371,13 @@
371371
"arguments": [0, {"var": "user"}],
372372
"url": "https://www.patreon.com/$user",
373373
"render": [{
374-
"match": "^patreon\\.com/api/posts",
374+
"match": "^patreon\\.com/api/posts.+campaign_id%5D=(\\d+)",
375+
"validate": ["campaignId"],
375376
"acceptJson": [
376377
{"op": "$.data.*", "var": "out:posts", "acceptJson": [
377-
{"var": "out:title", "op": "$.attributes.title"},
378-
{"var": "out:url", "op": "$.attributes.url"},
379-
{"var": "out:description", "op": "$.attributes.teaser_text"},
378+
{"var": "out:title", "op": "$.attributes.title", "mod": ["str"]},
379+
{"var": "out:url", "op": "$.attributes.url", "mod": ["url"]},
380+
{"var": "out:description", "op": "$.attributes.teaser_text", "mod": ["str"]},
380381
{"var": "out:publishedAt", "op": "$.attributes.published_at",
381382
"mod": ["date"]}
382383
]}
@@ -386,7 +387,7 @@
386387
{"op": "//script[contains(text(), 'window.patreon.bootstrap')]",
387388
"match": "window.patreon.bootstrap,\\s+({[\\s\\S]+?})\\);",
388389
"acceptJson": [
389-
{"var": "out:photos:avatar", "op": "$.campaign.data.attributes.avatar_photo_url"},
390+
{"var": "out:photos:avatar", "op": "$.campaign.data.attributes.avatar_photo_url", "mod": ["url"]},
390391
{"var": "campaignId", "op": "$.campaign.data.id"}
391392
]},
392393
{"var": "out:title", "op": "//meta[@name='title']/@content"},
@@ -405,11 +406,11 @@
405406
"url": "https://www.pinterest.com/resource/UnauthProfilePinFeedResource/get/?source_url=%2F$user%2F&data=%7B%22options%22%3A%7B%22isPrefetch%22%3Afalse%2C%22add_vase%22%3Atrue%2C%22field_set_key%22%3A%22unauth_react%22%2C%22page_size%22%3A25%2C%22username%22%3A%22$user%22%7D%2C%22context%22%3A%7B%7D%7D&_=1580963929964",
406407
"acceptJson": [
407408
{"op": "$.resource_response.data.*", "var": "out:posts", "acceptJson": [
408-
{"var": "out:title", "op": ["$.rich_metadata.title", "$.pin_join.seo_description"]},
409-
{"var": "out:text", "op": ["$.rich_metadata.description"]},
409+
{"var": "out:title", "op": ["$.rich_metadata.title", "$.pin_join.seo_description"], "mod": ["str"]},
410+
{"var": "out:text", "op": ["$.rich_metadata.description"], "mod": ["str"]},
410411
{"var": "out:publishedAt", "op": "$.created_at", "mod": ["date"]},
411-
{"var": "out:graphic:full", "op": "$.images.orig.url"},
412-
{"var": "out:url", "op": "$.link"}
412+
{"var": "out:graphic:full", "op": "$.images.orig.url", "mod": ["url"]},
413+
{"var": "out:url", "op": "$.link", "mod": ["url"]}
413414
]}
414415
]
415416
},
@@ -484,18 +485,18 @@
484485
"entry": [
485486
{"var": "id", "op": ["$.trail[0].post.id", "$.id_string"]},
486487
{"var": "author", "op": ["$.trail[0].blog.name", "$.blog_name"]},
487-
{"var": "out:title", "op": ["$.title", "$.summary", "$.date"]},
488+
{"var": "out:title", "op": ["$.title", "$.summary", "$.date"], "mod": ["str"]},
488489
{"var": "out:author", "op": "=$author"},
489-
{"var": "out:html", "op": "$.body"},
490+
{"var": "out:html", "op": "$.body", "mod": ["str"]},
490491
{"var": "out:url", "op": "=https://$author.tumblr.com/post/$id", "mod": ["url"]},
491492
{"var": "out:publishedAt", "op": "$.timestamp",
492493
"mod": ["int", "*1000", "date"]}
493494
]
494495
},
495496
"acceptJson": [
496-
{"var": "out:title", "op": "$.response.blog.title"},
497-
{"var": "out:author", "op": "$.response.blog.name"},
498-
{"var": "out:url", "op": "$.response.blog.url"},
497+
{"var": "out:title", "op": "$.response.blog.title", "mod": ["str"]},
498+
{"var": "out:author", "op": "$.response.blog.name", "mod": ["str"]},
499+
{"var": "out:url", "op": "$.response.blog.url", "mod": ["url"]},
499500
{"op": "$.response.blog.avatar.*", "acceptJson": [
500501
{"var": "width", "op": "$.width"},
501502
{"var": "height", "op": "$.height"},
@@ -518,28 +519,28 @@
518519
"arguments": [0, 0, {"var": "user"}],
519520
"url": "https://www.tiktok.com/$user",
520521
"render": [{
521-
"match": "^m\\.tiktok\\.com\\/api\\/(?:post\\/)?item_list.+&secUid=(\\w+)",
522+
"match": "^m\\.tiktok\\.com\\/api\\/(?:post\\/)?item_list.+&secUid=([-\\w]+)",
522523
"validate": ["secUid"],
523524
"acceptJson": [
524525
{"op": ["$.itemList.*", "$.items.*"], "var": "out:posts", "acceptJson": [
525-
{"var": "out:title", "op": "$.desc"},
526+
{"var": "out:title", "op": "$.desc", "mod": ["str"]},
526527
{"var": "itemId", "op": "$.id"},
527528
{"var": "username", "op": "$.author.uniqueId"},
528529
{"var": "out:publishedAt", "op": "$.createTime",
529530
"mod": ["int", "*1000", "date"]},
530-
{"var": "out:graphic:full", "op": ["$.video.dynamicCover", "$.video.cover"]},
531+
{"var": "out:graphic:full", "op": ["$.video.dynamicCover", "$.video.cover"], "mod": ["url"]},
531532
{"var": "out:url", "op": "=https://www.tiktok.com/@$username/video/$itemId"}
532533
]}
533534
]
534535
}],
535536
"acceptHtml": [
536537
{"op": "//script[@id='__NEXT_DATA__']/text()", "match": ".", "acceptJson": [
538+
{"var": "secUid", "op": ["$.props.pageProps.userInfo.user.secUid", "$.props.pageProps.feedConfig.secUid"]},
537539
{"op": "$.props.pageProps.userInfo.user", "acceptJson": [
538540
{"var": "userId", "op": "$.id"},
539-
{"var": "secUid", "op": "$.secUid"},
540-
{"var": "out:title", "op": "$.nickname"},
541-
{"var": "out:photos:avatar", "op": "$.avatarThumb"},
542-
{"var": "out:description", "op": "$.signature"}
541+
{"var": "out:title", "op": "$.nickname", "mod": ["str"]},
542+
{"var": "out:photos:avatar", "op": "$.avatarThumb", "mod": ["url"]},
543+
{"var": "out:description", "op": "$.signature", "mod": ["str"]}
543544
]}
544545
]}
545546
]
@@ -551,9 +552,9 @@
551552
"acceptJson": [
552553
{"op": "$.data[?(@.login === '$streamer')]", "acceptJson": [
553554
{"var": "userId", "op": "$.id"},
554-
{"var": "out:title", "op": "$.display_name"},
555-
{"var": "out:photos:avatar", "op": "$.profile_image_url"},
556-
{"var": "out:description", "op": "$.description"}
555+
{"var": "out:title", "op": "$.display_name", "mod": ["str"]},
556+
{"var": "out:photos:avatar", "op": "$.profile_image_url", "mod": ["url"]},
557+
{"var": "out:description", "op": "$.description", "mod": ["str"]}
557558
]}
558559
]
559560
},
@@ -563,9 +564,9 @@
563564
"url": "https://api.twitch.tv/helix/streams",
564565
"acceptJson": [
565566
{"op": "$.data[?(@.user_id === '$userId')]", "var": "out:status", "acceptJson": [
566-
{"var": "out:type", "op": "$.type"},
567-
{"var": "out:title", "op": "$.title"},
568-
{"var": "out:graphic:thumb", "op": "$.thumbnail_url"},
567+
{"var": "out:type", "op": "$.type", "mod": ["str"]},
568+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
569+
{"var": "out:graphic:thumb", "op": "$.thumbnail_url", "mod": ["url"]},
569570
{"var": "out:publishedAt", "op": "$.started_at", "mod": ["date"]}
570571
]}
571572
]
@@ -579,10 +580,10 @@
579580
"url": "https://api.twitch.tv/helix/videos?type=archive&user_id=$userId",
580581
"acceptJson": [
581582
{"op": "$.data.*", "var": "out:posts", "acceptJson": [
582-
{"var": "out:title", "op": "$.title"},
583-
{"var": "out:text", "op": "$.description"},
584-
{"var": "out:url", "op": "$.url"},
585-
{"var": "out:graphic:thumb", "op": "$.thumbnail_url"},
583+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
584+
{"var": "out:text", "op": "$.description", "mod": ["str"]},
585+
{"var": "out:url", "op": "$.url", "mod": ["url"]},
586+
{"var": "out:graphic:thumb", "op": "$.thumbnail_url", "mod": ["url"]},
586587
{"var": "out:publishedAt", "op": "$.published_at", "mod": ["date"]}
587588
]}
588589
]
@@ -601,12 +602,12 @@
601602
"url": "https://api.twitter.com/1.1/users/show.json?screen_name=$username",
602603
"acceptJson": [
603604
{"var": "userId", "op": "$.id_str"},
604-
{"var": "out:author", "op": "$.screen_name"},
605-
{"var": "out:title", "op": "$.name"},
606-
{"var": "out:photos:avatar", "op": "$.profile_image_url"},
607-
{"var": "out:description", "op": "$.description"},
605+
{"var": "out:author", "op": "$.screen_name", "mod": ["str"]},
606+
{"var": "out:title", "op": "$.name", "mod": ["str"]},
607+
{"var": "out:photos:avatar", "op": "$.profile_image_url", "mod": ["url"]},
608+
{"var": "out:description", "op": "$.description", "mod": ["str"]},
608609
{"op": "$.entities.*.urls.*", "var": "out:rels", "acceptJson": [
609-
{"var": "out:url", "op": "$.expanded_url"}
610+
{"var": "out:url", "op": "$.expanded_url", "mod": ["url"]}
610611
]}
611612
]
612613
},
@@ -686,8 +687,8 @@
686687
{"op": "$.contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents.*",
687688
"var": "out:posts", "acceptJson": [
688689
{"var": "videoId", "op": "$.playlistVideoRenderer.videoId"},
689-
{"var": "out:graphic:full", "op": "$.playlistVideoRenderer.thumbnail.thumbnails[-1:].url"},
690-
{"var": "out:title", "op": ["$.playlistVideoRenderer.title.simpleText", "$.playlistVideoRenderer.title.runs[0].text"]},
690+
{"var": "out:graphic:full", "op": "$.playlistVideoRenderer.thumbnail.thumbnails[-1:].url", "mod": ["url"]},
691+
{"var": "out:title", "op": ["$.playlistVideoRenderer.title.simpleText", "$.playlistVideoRenderer.title.runs[0].text"], "mod": ["str"]},
691692
{"var": "out:url", "op": "=https://www.youtube.com/watch?v=$videoId&list=$id"}
692693
]}
693694
]}
@@ -748,8 +749,8 @@
748749
{"var": "out:url", "op": "=https://futureland.tv/$user/$slug/$id"},
749750
{"var": "out:publishedAt", "op": "$.createdAt"},
750751
{"var": "out:updatedAt", "op": "$.updatedAt"},
751-
{"var": "out:graphic:thumb", "op": "$.preview_link"},
752-
{"var": "out:text", "op": "$.notes"}
752+
{"var": "out:graphic:thumb", "op": "$.preview_link", "mod": ["url"]},
753+
{"var": "out:text", "op": "$.notes", "mod": ["str"]}
753754
]}
754755
]
755756
},
@@ -759,17 +760,17 @@
759760
"arguments": [0, {"var": "user"}, {"var": "journal"}],
760761
"url": "https://futureland.tv/api/users/$user/journals/$journal",
761762
"acceptJson": [
762-
{"var": "out:title", "op": "$.title"},
763-
{"var": "out:photos:avatar", "op": "$.iconLink"},
764-
{"var": "out:description", "op": "$.description"},
763+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
764+
{"var": "out:photos:avatar", "op": "$.iconLink", "mod": ["url"]},
765+
{"var": "out:description", "op": "$.description", "mod": ["str"]},
765766
{"op": "$.entries.*", "var": "out:posts", "acceptJson": [
766767
{"var": "id", "op": "$.id"},
767768
{"var": "author", "op": "$.user.futureland_user"},
768769
{"var": "out:url", "op": "=https://futureland.tv/$author/$journal/$id"},
769770
{"var": "out:publishedAt", "op": "$.createdAt"},
770771
{"var": "out:updatedAt", "op": "$.updatedAt"},
771-
{"var": "out:graphic:thumb", "op": "$.preview_link"},
772-
{"var": "out:text", "op": "$.notes"}
772+
{"var": "out:graphic:thumb", "op": "$.preview_link", "mod": ["url"]},
773+
{"var": "out:text", "op": "$.notes", "mod": ["str"]}
773774
]}
774775
]
775776
},
@@ -796,7 +797,7 @@
796797
{"var": "out:photos:avatar", "op": "$.props.pageProps.account.profilePictureUrl"},
797798
{"op": "$.props.pageProps.account.links", "var": "out:rels", "acceptJson": [
798799
{"var": "out:url", "op": "$.url", "mod": ["url"]},
799-
{"var": "out:title", "op": "$.title"}
800+
{"var": "out:title", "op": "$.title", "mod": ["str"]}
800801
]}
801802
]}
802803
]
@@ -853,8 +854,8 @@
853854
"store.steampowered.com:app": {
854855
"url": "https://store.steampowered.com/api/appdetails?appids=$app",
855856
"acceptJson": [
856-
{"var": "out:title", "op": "$.$app.data.name"},
857-
{"var": "out:description", "op": "$.$app.data.short_description"}
857+
{"var": "out:title", "op": "$.$app.data.name", "mod": ["str"]},
858+
{"var": "out:description", "op": "$.$app.data.short_description", "mod": ["str"]}
858859
]
859860
},
860861

@@ -885,8 +886,8 @@
885886
{"var": "out:url", "op": "$.url", "mod": ["url"]},
886887
{"var": "out:publishedAt", "op": "$.date",
887888
"mod": ["int", "*1000", "date"]},
888-
{"var": "out:title", "op": "$.title"},
889-
{"var": "out:html", "op": "$.contents"}
889+
{"var": "out:title", "op": "$.title", "mod": ["str"]},
890+
{"var": "out:html", "op": "$.contents", "mod": ["str"]}
890891
]}
891892
]
892893
}

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@
8181
"electron-updater": "^4.3.8",
8282
"ent": "^2.2.0",
8383
"fast-json-patch": "^3.0.0-1",
84-
"fraidyscrape": "^1.0.12",
84+
"fraidyscrape": "^1.0.14",
8585
"fsbin": "^1.0.9",
8686
"hyperapp": "^1.2.9",
8787
"json-date-parser": "^1.0.1",

‎src/js/util.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ export function urlToID (normLink) {
7070
}
7171

7272
export function followTitle(follow) {
73-
return follow.title || follow.actualTitle || follow.url
73+
return (follow.title || follow.actualTitle || follow.url).toString()
7474
}
7575

7676
export function isValidFollow(follow) {

0 commit comments

Comments
 (0)
Please sign in to comment.