Skip to content

Fixes building the database from scratch in twitter #691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SaptakS wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fixes building the database from scratch in twitter #691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5c63e74
X: Increases sleep time to load content and DOM
SaptakS Aug 6, 2026
c350221
X: Removes legacy object path
SaptakS Aug 6, 2026
9c7dfec
X: Updates path for DM from /messages to /i/chat
SaptakS Aug 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions src/account_x/controller/index/indexParseTweetsResponseData.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,6 @@ export function indexParseTweetsResponseData(
.user_results &&
entries.content.itemContent.tweet_results.result.core.user_results
.result &&
entries.content.itemContent.tweet_results.result.core.user_results
.result.legacy &&
entries.content.itemContent.tweet_results.result.core.user_results
.result.core &&
entries.content.itemContent.tweet_results.result.legacy
Expand All @@ -171,8 +169,6 @@ export function indexParseTweetsResponseData(
.user_results &&
entries.content.itemContent.tweet_results.result.tweet.core
.user_results.result &&
entries.content.itemContent.tweet_results.result.tweet.core
.user_results.result.legacy &&
entries.content.itemContent.tweet_results.result.tweet.core
.user_results.result.core &&
entries.content.itemContent.tweet_results.result.tweet.legacy
Expand Down
2 changes: 1 addition & 1 deletion src/renderer/src/view_models/XViewModel/jobs_delete.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -810,7 +810,7 @@ describe("jobs_delete.ts", () => {

// Should load DMs page at least once
expect(vm.loadURLWithRateLimit).toHaveBeenCalledWith(
"https://x.com/messages",
"https://x.com/i/chat",
);
});

Expand Down
6 changes: 3 additions & 3 deletions src/renderer/src/view_models/XViewModel/jobs_delete/helpers_pages.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ describe("helpers_pages.ts", () => {
await DeleteHelpers.deleteDMsLoadDMsPage(vm);

expect(vm.loadURLWithRateLimit).toHaveBeenCalledWith(
"https://x.com/messages",
"https://x.com/i/chat",
);
});

Expand All @@ -48,7 +48,7 @@ describe("helpers_pages.ts", () => {

expect(vm.waitForSelector).toHaveBeenCalledWith(
'section input[type="text"]',
"https://x.com/messages",
"https://x.com/i/chat",
30000,
);
});
Expand All @@ -58,7 +58,7 @@ describe("helpers_pages.ts", () => {

expect(vm.waitForSelector).toHaveBeenCalledWith(
'section div div[role="tablist"] div[data-testid="cellInnerDiv"]',
"https://x.com/messages",
"https://x.com/i/chat",
);
});

Expand Down
6 changes: 3 additions & 3 deletions src/renderer/src/view_models/XViewModel/jobs_delete/helpers_pages.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ export async function deleteDMsLoadDMsPage(vm: XViewModel): Promise<boolean> {

success = false;
for (tries = 0; tries < 3; tries++) {
await vm.loadURLWithRateLimit("https://x.com/messages");
await vm.loadURLWithRateLimit("https://x.com/i/chat");

// If the conversations list is empty, there is no search text field
try {
// Wait for the search text field to appear with a 30 second timeout
await vm.waitForSelector(
'section input[type="text"]',
"https://x.com/messages",
"https://x.com/i/chat",
30000,
);
} catch (e) {
Expand All @@ -44,7 +44,7 @@ export async function deleteDMsLoadDMsPage(vm: XViewModel): Promise<boolean> {
);
await vm.waitForSelector(
'section div div[role="tablist"] div[data-testid="cellInnerDiv"]',
"https://x.com/messages",
"https://x.com/i/chat",
);
success = true;
break;
Expand Down
8 changes: 4 additions & 4 deletions src/renderer/src/view_models/XViewModel/jobs_index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ describe("jobs_index.ts", () => {
vi.spyOn(vm, "waitForSelector")
.mockResolvedValueOnce(undefined) // search field
.mockRejectedValueOnce(
new URLChangedError("https://x.com/messages", "https://x.com/other"),
new URLChangedError("https://x.com/i/chat", "https://x.com/other"),
);

const result = await IndexJobs.runJobIndexConversations(vm, 0);
Expand Down Expand Up @@ -629,10 +629,10 @@ describe("jobs_index.ts", () => {
await IndexJobs.runJobIndexMessages(vm, 0);

expect(vm.loadURLWithRateLimit).toHaveBeenCalledWith(
"https://x.com/messages/conv1",
"https://x.com/i/chat/conv1",
);
expect(vm.loadURLWithRateLimit).toHaveBeenCalledWith(
"https://x.com/messages/conv2",
"https://x.com/i/chat/conv2",
);
expect(mockElectron.X.indexConversationFinished).toHaveBeenCalledTimes(2);
});
Expand Down Expand Up @@ -668,7 +668,7 @@ describe("jobs_index.ts", () => {
mockElectron.X.indexMessagesStart.mockResolvedValue(mockMessagesData);
vi.spyOn(vm, "waitForSelector").mockRejectedValue(
new URLChangedError(
"https://x.com/messages/conv1",
"https://x.com/i/chat/conv1",
"https://x.com/i/verified-get-verified",
),
);
Expand Down
14 changes: 7 additions & 7 deletions src/renderer/src/view_models/XViewModel/jobs_index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export async function runJobIndexTweets(
const username = vm.account.xAccount?.username || "";
const url = `https://x.com/${username}/with_replies`;
await vm.loadURLWithRateLimit(url);
await vm.sleep(500);
await vm.sleep(2000);

// Check if tweets list is empty
if (
Expand Down Expand Up @@ -202,7 +202,7 @@ export async function runJobIndexConversations(
await window.electron.X.resetRateLimitInfo(vm.account.id);

// Load the messages page
await vm.loadURLWithRateLimit("https://x.com/messages");
await vm.loadURLWithRateLimit("https://x.com/i/chat");
vm.log(
"runJobIndexConversations",
"loaded messages page, waiting for conversations to load",
Expand All @@ -220,7 +220,7 @@ export async function runJobIndexConversations(
// Wait for the search text field to appear with a 30 second timeout
await vm.waitForSelector(
'section input[type="text"]',
"https://x.com/messages",
"https://x.com/i/chat",
30000,
);
} catch (e) {
Expand All @@ -237,7 +237,7 @@ export async function runJobIndexConversations(
try {
await vm.waitForSelector(
'section div div[role="tablist"] div[data-testid="cellInnerDiv"]',
"https://x.com/messages",
"https://x.com/i/chat",
);
break;
} catch (e) {
Expand Down Expand Up @@ -405,7 +405,7 @@ export async function runJobIndexMessages(vm: XViewModel, jobIndex: number) {

// Load URL and wait for messages to appear
try {
url = `https://x.com/messages/${indexMessagesStartResponse.conversationIDs[i]}`;
url = `https://x.com/i/chat/${indexMessagesStartResponse.conversationIDs[i]}`;
await vm.loadURLWithRateLimit(url);
// Use longer timeout on retries (60 seconds instead of default 30 seconds)
const timeout = tries > 0 ? 60000 : undefined;
Expand Down Expand Up @@ -568,7 +568,7 @@ export async function runJobIndexLikes(
const username = vm.account.xAccount?.username || "";
const url = `https://x.com/${username}/likes`;
await vm.loadURLWithRateLimit(url);
await vm.sleep(500);
await vm.sleep(2000);

// Check if likes list is empty
if (
Expand Down Expand Up @@ -665,7 +665,7 @@ export async function runJobIndexBookmarks(
await window.electron.X.resetRateLimitInfo(vm.account.id);
const url = "https://x.com/i/bookmarks";
await vm.loadURLWithRateLimit(url);
await vm.sleep(500);
await vm.sleep(2000);

// Check if bookmarks list is empty
if (
Expand Down
Loading