Skip to content

Commit

Permalink
Use search instead of old timeline endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
@zedeus
zedeus committed Jul 22, 2023
1 parent cc5841d commit 50f821d
Show file tree
Hide file tree
Showing 7 changed files with 180 additions and 167 deletions.
28 changes: 15 additions & 13 deletions src/api.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after)

proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = oldUserTweets / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
# if id.len == 0: return
# let
# ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
# url = oldUserTweets / (id & ".json") ? ps
# result = parseTimeline(await fetch(url, Api.timeline), after)

proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
Expand Down Expand Up @@ -123,20 +123,22 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
result.tweets.query = query

proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
let q = genQueryParam(query)
var q = genQueryParam(query)

if q.len == 0 or q == emptyQuery:
return Timeline(query: query, beginning: true)

if after.len > 0:
q &= " max_id:" & after

let url = tweetSearch ? genParams({
"q": q,
"tweet_search_mode": "live",
"max_id": after
"q": q ,
"modules": "status",
"result_type": "recent",
})

result = parseTweetSearch(await fetch(url, Api.search))
result = parseTweetSearch(await fetch(url, Api.search), after)
result.query = query
if after.len == 0:
result.beginning = true

proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
if query.text.len == 0:
Expand Down
2 changes: 1 addition & 1 deletion src/apiutils.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
for p in pars:
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified")
result &= ("include_ext_alt_text", "1")
result &= ("include_ext_media_availability", "1")
if count.len > 0:
Expand Down
55 changes: 28 additions & 27 deletions src/consts.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ const

photoRail* = api / "1.1/statuses/media_timeline.json"
userSearch* = api / "1.1/users/search.json"
tweetSearch* = api / "1.1/search/tweets.json"
tweetSearch* = api / "1.1/search/universal.json"

oldUserTweets* = api / "2/timeline/profile"
# oldUserTweets* = api / "2/timeline/profile"

graphql = api / "graphql"
graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
Expand All @@ -28,27 +28,28 @@ const
graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"

timelineParams* = {
"include_profile_interstitial_type": "0",
"include_blocking": "0",
"cards_platform": "Web-13",
"tweet_mode": "extended",
"ui_lang": "en-US",
"send_error_codes": "1",
"simple_quoted_tweet": "1",
"skip_status": "1",
"include_blocked_by": "0",
"include_followed_by": "0",
"include_want_retweets": "0",
"include_mute_edge": "0",
"include_blocking": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_ext_is_blue_verified": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "0",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "1",
"include_user_entities": "1",
"include_ext_is_blue_verified": "1",
"include_ext_media_color": "0",
"send_error_codes": "1",
"simple_quoted_tweet": "1",
"include_quote_count": "1"
"include_followed_by": "0",
"include_mute_edge": "0",
"include_profile_interstitial_type": "0",
"include_quote_count": "1",
"include_reply_count": "1",
"include_user_entities": "1",
"include_want_retweets": "0",
}.toSeq

gqlFeatures* = """{
Expand Down Expand Up @@ -100,17 +101,17 @@ const
"includeHasBirdwatchNotes": false
}"""

oldUserTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withV2Timeline": true
}
"""
# oldUserTweetsVariables* = """{
# "userId": "$1", $2
# "count": 20,
# "includePromotedContent": false,
# "withDownvotePerspective": false,
# "withReactionsMetadata": false,
# "withReactionsPerspective": false,
# "withVoice": false,
# "withV2Timeline": true
# }
# """

userTweetsVariables* = """{
"rest_id": "$1", $2
Expand Down
206 changes: 104 additions & 102 deletions src/parser.nim
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, tables, times, math
import strutils, options, times, math
import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
Expand Down Expand Up @@ -295,110 +295,112 @@ proc parseLegacyTweet(js: JsonNode): Tweet =
if result.quote.isSome:
result.quote = some parseLegacyTweet(js{"quoted_status"})

proc parseTweetSearch*(js: JsonNode): Timeline =
if js.kind == JNull or "statuses" notin js:
return Timeline(beginning: true)
proc parseTweetSearch*(js: JsonNode; after=""): Timeline =
result.beginning = after.len == 0

for tweet in js{"statuses"}:
let parsed = parseLegacyTweet(tweet)

if parsed.retweet.isSome:
parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})

result.content.add @[parsed]

let cursor = js{"search_metadata", "next_results"}.getStr
if cursor.len > 0 and "max_id" in cursor:
result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")]

proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
let intId = if id.len > 0: parseBiggestInt(id) else: 0
result = global.tweets.getOrDefault(id, Tweet(id: intId))

if result.quote.isSome:
let quote = get(result.quote).id
if $quote in global.tweets:
result.quote = some global.tweets[$quote]
else:
result.quote = some Tweet()

if result.retweet.isSome:
let rt = get(result.retweet).id
if $rt in global.tweets:
result.retweet = some finalizeTweet(global, $rt)
else:
result.retweet = some Tweet()

proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return

let id = pin.getId
if id notin global.tweets: return

global.tweets[id].pinned = true
return finalizeTweet(global, id)

proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects()
let
tweets = ? js{"globalObjects", "tweets"}
users = ? js{"globalObjects", "users"}

for k, v in users:
result.users[k] = parseUser(v, k)

for k, v in tweets:
var tweet = parseTweet(v, v{"card"})
if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet

proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0:
if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0:
return

for i in js:
if res.tweets.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.pinned = some pin

with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr:
res.tweets.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr:
res.tweets.bottom = r.getCursor

proc parseTimeline*(js: JsonNode; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0))
let global = parseGlobalObjects(? js)

let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0: return

result.parseInstructions(global, instructions)

var entries: JsonNode
for i in instructions:
if "addEntries" in i:
entries = i{"addEntries", "entries"}

for e in ? entries:
let entry = e{"entryId"}.getStr
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue
result.tweets.content.add tweet
elif "cursor-top" in entry:
result.tweets.top = e.getCursor
elif "cursor-bottom" in entry:
result.tweets.bottom = e.getCursor
elif entry.startsWith("sq-cursor"):
with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom":
result.tweets.bottom = cursor{"value"}.getStr
else:
result.tweets.top = cursor{"value"}.getStr
for item in js{"modules"}:
with tweet, item{"status", "data"}:
let parsed = parseLegacyTweet(tweet)

if parsed.retweet.isSome:
parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})

result.content.add @[parsed]

if result.content.len > 0:
result.bottom = $(result.content[^1][0].id - 1)

# proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
# let intId = if id.len > 0: parseBiggestInt(id) else: 0
# result = global.tweets.getOrDefault(id, Tweet(id: intId))

# if result.quote.isSome:
# let quote = get(result.quote).id
# if $quote in global.tweets:
# result.quote = some global.tweets[$quote]
# else:
# result.quote = some Tweet()

# if result.retweet.isSome:
# let rt = get(result.retweet).id
# if $rt in global.tweets:
# result.retweet = some finalizeTweet(global, $rt)
# else:
# result.retweet = some Tweet()

# proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
# let pin = js{"pinEntry", "entry", "entryId"}.getStr
# if pin.len == 0: return

# let id = pin.getId
# if id notin global.tweets: return

# global.tweets[id].pinned = true
# return finalizeTweet(global, id)

# proc parseGlobalObjects(js: JsonNode): GlobalObjects =
# result = GlobalObjects()
# let
# tweets = ? js{"globalObjects", "tweets"}
# users = ? js{"globalObjects", "users"}

# for k, v in users:
# result.users[k] = parseUser(v, k)

# for k, v in tweets:
# var tweet = parseTweet(v, v{"card"})
# if tweet.user.id in result.users:
# tweet.user = result.users[tweet.user.id]
# result.tweets[k] = tweet

# proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
# if js.kind != JArray or js.len == 0:
# return

# for i in js:
# if res.tweets.beginning and i{"pinEntry"}.notNull:
# with pin, parsePin(i, global):
# res.pinned = some pin

# with r, i{"replaceEntry", "entry"}:
# if "top" in r{"entryId"}.getStr:
# res.tweets.top = r.getCursor
# elif "bottom" in r{"entryId"}.getStr:
# res.tweets.bottom = r.getCursor

# proc parseTimeline*(js: JsonNode; after=""): Profile =
# result = Profile(tweets: Timeline(beginning: after.len == 0))
# let global = parseGlobalObjects(? js)

# let instructions = ? js{"timeline", "instructions"}
# if instructions.len == 0: return

# result.parseInstructions(global, instructions)

# var entries: JsonNode
# for i in instructions:
# if "addEntries" in i:
# entries = i{"addEntries", "entries"}

# for e in ? entries:
# let entry = e{"entryId"}.getStr
# if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
# let tweet = finalizeTweet(global, e.getEntryId)
# if not tweet.available: continue
# result.tweets.content.add tweet
# elif "cursor-top" in entry:
# result.tweets.top = e.getCursor
# elif "cursor-bottom" in entry:
# result.tweets.bottom = e.getCursor
# elif entry.startsWith("sq-cursor"):
# with cursor, e{"content", "operation", "cursor"}:
# if cursor{"cursorType"}.getStr == "Bottom":
# result.tweets.bottom = cursor{"value"}.getStr
# else:
# result.tweets.top = cursor{"value"}.getStr

proc parsePhotoRail*(js: JsonNode): PhotoRail =
with error, js{"error"}:
Expand Down
12 changes: 10 additions & 2 deletions src/routes/timeline.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,26 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;

result =
case query.kind
of posts: await getTimeline(userId, after)
# of posts: await getTimeline(userId, after)
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
of media: await getGraphUserTweets(userId, TimelineKind.media, after)
else: Profile(tweets: await getTweetSearch(query, after))

result.user = await user
result.photoRail = await rail

result.tweets.query = query

if result.user.protected or result.user.suspended:
return

result.tweets.query = query
if not skipPinned and query.kind == posts and
result.user.pinnedTweet > 0 and after.len == 0:
let tweet = await getCachedTweet(result.user.pinnedTweet)
if not tweet.isNil:
tweet.pinned = true
tweet.user = result.user
result.pinned = some tweet

proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} =
Expand Down
Loading

0 comments on commit 50f821d

Please sign in to comment.