Improve seller account filtering
This commit is contained in:
1 parent
d819a0f1e6
commit
f4e8fdc42f
5 files changed
+178
-52
No files matched your search
@@ -254,29 +254,6 @@ export async function runMainLoop(app: AppContainer): Promise<void> {
|
|||||||
processedAt: Date.now(),
|
processedAt: Date.now(),
|
||||||
});
|
});
|
||||||
|
|
||||||
// Skip OF/seller accounts before spending on AI.
|
|
||||||
if (
|
|
||||||
config.reddit.filterSellers &&
|
|
||||||
(config.reddit.sellerKeywords.length > 0 || config.reddit.sellerDomains.length > 0)
|
|
||||||
) {
|
|
||||||
const isSeller = await redditApi.isSellerProfile(
|
|
||||||
post.author,
|
|
||||||
config.reddit.sellerKeywords,
|
|
||||||
config.reddit.sellerDomains
|
|
||||||
);
|
|
||||||
if (isSeller) {
|
|
||||||
logger.info(`Skipping u/${post.author} - seller/OnlyFans profile detected`);
|
|
||||||
webServer?.logActivity(
|
|
||||||
'skipped',
|
|
||||||
`Skipped post by u/${post.author} (seller/OnlyFans profile)`,
|
|
||||||
post.author,
|
|
||||||
);
|
|
||||||
webServer?.pushLog('info', `🚫 Skipped u/${post.author} - seller/OnlyFans profile`);
|
|
||||||
postsRepo.setError(post.id, 'Seller profile detected');
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Refresh examples weighted toward this post's subreddit.
|
// Refresh examples weighted toward this post's subreddit.
|
||||||
const subredditExamples = responseChecker.getSuccessfulExamples(3, post.subreddit);
|
const subredditExamples = responseChecker.getSuccessfulExamples(3, post.subreddit);
|
||||||
processorService.setSuccessfulExamples(subredditExamples);
|
processorService.setSuccessfulExamples(subredditExamples);
|
||||||
|
|||||||
+95
-29
@@ -48,6 +48,38 @@ export interface RedditPost {
|
|||||||
link_flair_text?: string | null;
|
link_flair_text?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface RedditListingChild {
|
||||||
|
data?: {
|
||||||
|
title?: unknown;
|
||||||
|
selftext?: unknown;
|
||||||
|
body?: unknown;
|
||||||
|
url?: unknown;
|
||||||
|
subreddit?: unknown;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function joinText(values: unknown[]): string {
|
||||||
|
return values.filter((value): value is string => typeof value === 'string').join(' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Match configured seller signals in user-controlled text. */
|
||||||
|
export function findSellerSignal(
|
||||||
|
text: string,
|
||||||
|
keywords: string[],
|
||||||
|
domains: string[],
|
||||||
|
): { kind: 'keyword' | 'domain'; value: string } | null {
|
||||||
|
const normalized = text.toLocaleLowerCase();
|
||||||
|
for (const keyword of keywords) {
|
||||||
|
const candidate = keyword.trim().toLocaleLowerCase();
|
||||||
|
if (candidate && normalized.includes(candidate)) {
|
||||||
|
return { kind: 'keyword', value: keyword };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const domain = findSellerDomain(text, domains);
|
||||||
|
return domain ? { kind: 'domain', value: domain } : null;
|
||||||
|
}
|
||||||
|
|
||||||
// Extracts URL-ish tokens (with or without scheme) and returns the first
|
// Extracts URL-ish tokens (with or without scheme) and returns the first
|
||||||
// domain match. Matches `host` or any subdomain (`*.host`), case-insensitive.
|
// domain match. Matches `host` or any subdomain (`*.host`), case-insensitive.
|
||||||
export function findSellerDomain(text: string, domains: string[]): string | null {
|
export function findSellerDomain(text: string, domains: string[]): string | null {
|
||||||
@@ -249,10 +281,28 @@ export class RedditApiClient {
|
|||||||
async isSellerProfile(
|
async isSellerProfile(
|
||||||
username: string,
|
username: string,
|
||||||
sellerKeywords: string[],
|
sellerKeywords: string[],
|
||||||
sellerDomains: string[] = []
|
sellerDomains: string[] = [],
|
||||||
|
currentPost?: RedditPost,
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
if (sellerKeywords.length === 0 && sellerDomains.length === 0) return false;
|
if (sellerKeywords.length === 0 && sellerDomains.length === 0) return false;
|
||||||
|
|
||||||
|
// Check the post first. This catches direct promotional posts without an
|
||||||
|
// extra request and also covers accounts whose profile bio is deliberately
|
||||||
|
// vague.
|
||||||
|
if (currentPost) {
|
||||||
|
const postSignal = findSellerSignal(
|
||||||
|
joinText([currentPost.title, currentPost.selftext, currentPost.url]),
|
||||||
|
sellerKeywords,
|
||||||
|
sellerDomains,
|
||||||
|
);
|
||||||
|
if (postSignal) {
|
||||||
|
this.logger.info(
|
||||||
|
`u/${username} flagged as seller from current post (matched ${postSignal.kind}: "${postSignal.value}")`,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const cookieHeader = await this.getCookieHeader();
|
const cookieHeader = await this.getCookieHeader();
|
||||||
const headers: Record<string, string> = {};
|
const headers: Record<string, string> = {};
|
||||||
@@ -260,51 +310,67 @@ export class RedditApiClient {
|
|||||||
headers['Cookie'] = cookieHeader;
|
headers['Cookie'] = cookieHeader;
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await this.client.get(`/user/${username}/about.json`, {
|
const response = await this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
|
||||||
params: { raw_json: 1 },
|
params: { raw_json: 1 },
|
||||||
headers,
|
headers,
|
||||||
});
|
});
|
||||||
|
|
||||||
const userData = response.data?.data;
|
const userData = response.data?.data;
|
||||||
if (!userData) return false;
|
if (userData) {
|
||||||
|
const profileText = joinText([
|
||||||
// Combine all text fields to scan
|
userData.subreddit?.public_description || '',
|
||||||
const textToCheck = [
|
userData.subreddit?.title || '',
|
||||||
userData.subreddit?.public_description || '',
|
userData.subreddit?.display_name_prefixed || '',
|
||||||
userData.subreddit?.title || '',
|
]);
|
||||||
userData.subreddit?.display_name_prefixed || '',
|
const profileSignal = findSellerSignal(profileText, sellerKeywords, sellerDomains);
|
||||||
]
|
if (profileSignal) {
|
||||||
.join(' ')
|
|
||||||
.toLowerCase();
|
|
||||||
|
|
||||||
for (const keyword of sellerKeywords) {
|
|
||||||
if (textToCheck.includes(keyword.toLowerCase())) {
|
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`u/${username} flagged as seller profile (matched keyword: "${keyword}")`
|
`u/${username} flagged as seller profile (matched ${profileSignal.kind}: "${profileSignal.value}")`,
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sellerDomains.length > 0) {
|
|
||||||
const matchedDomain = findSellerDomain(textToCheck, sellerDomains);
|
|
||||||
if (matchedDomain) {
|
|
||||||
this.logger.info(
|
|
||||||
`u/${username} flagged as seller profile (matched domain: "${matchedDomain}")`
|
|
||||||
);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (axios.isAxiosError(error) && error.response?.status === 404) {
|
if (axios.isAxiosError(error) && error.response?.status === 404) {
|
||||||
this.logger.warn(`User u/${username} profile not found`);
|
this.logger.warn(`User u/${username} profile not found`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
this.logger.warn(`Failed to check profile for u/${username}:`, error);
|
this.logger.warn(`Failed to check profile for u/${username}:`, error);
|
||||||
return false; // Don't block on errors - let it through
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Bios are often clean while recent posts advertise the paid account.
|
||||||
|
// Scan a small overview page, including comment bodies and subreddit
|
||||||
|
// names, before allowing the candidate through.
|
||||||
|
try {
|
||||||
|
const cookieHeader = await this.getCookieHeader();
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
||||||
|
const response = await this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
|
||||||
|
params: { limit: 25, raw_json: 1 },
|
||||||
|
headers,
|
||||||
|
});
|
||||||
|
const children: RedditListingChild[] = Array.isArray(response.data?.data?.children)
|
||||||
|
? response.data.data.children
|
||||||
|
: [];
|
||||||
|
const activityText = children.map(({ data }) => joinText([
|
||||||
|
data?.title,
|
||||||
|
data?.selftext,
|
||||||
|
data?.body,
|
||||||
|
data?.url,
|
||||||
|
data?.subreddit,
|
||||||
|
])).join(' ');
|
||||||
|
const activitySignal = findSellerSignal(activityText, sellerKeywords, sellerDomains);
|
||||||
|
if (activitySignal) {
|
||||||
|
this.logger.info(
|
||||||
|
`u/${username} flagged as seller from recent activity (matched ${activitySignal.kind}: "${activitySignal.value}")`,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(`Failed to check recent activity for u/${username}:`, error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return false; // Fail open if Reddit profile endpoints are unavailable.
|
||||||
}
|
}
|
||||||
|
|
||||||
getPostUrl(post: RedditPost): string {
|
getPostUrl(post: RedditPost): string {
|
||||||
|
|||||||
@@ -48,6 +48,32 @@ export class MonitorService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
this.config.reddit.filterSellers &&
|
||||||
|
(this.config.reddit.sellerKeywords.length > 0 || this.config.reddit.sellerDomains.length > 0)
|
||||||
|
) {
|
||||||
|
const isSeller = await this.redditApi.isSellerProfile(
|
||||||
|
post.author,
|
||||||
|
this.config.reddit.sellerKeywords,
|
||||||
|
this.config.reddit.sellerDomains,
|
||||||
|
post,
|
||||||
|
);
|
||||||
|
if (isSeller) {
|
||||||
|
this.logger.info(`Skipping post ${post.id} from u/${post.author} - seller signals detected`);
|
||||||
|
this.postsRepo.insert({
|
||||||
|
id: post.id,
|
||||||
|
subreddit: post.subreddit,
|
||||||
|
author: post.author,
|
||||||
|
title: post.title,
|
||||||
|
postUrl: this.redditApi.getPostUrl(post),
|
||||||
|
imageUrl: subreddit.type === 'image' ? this.redditApi.getImageUrl(post) : null,
|
||||||
|
processedAt: Date.now(),
|
||||||
|
});
|
||||||
|
this.postsRepo.setError(post.id, 'Seller signals detected');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Chat subs almost always have a self-declared gender tag in the
|
// Chat subs almost always have a self-declared gender tag in the
|
||||||
// title/body (e.g. [25F], M4F). Drop posts that don't match the
|
// title/body (e.g. [25F], M4F). Drop posts that don't match the
|
||||||
// configured targetGenders here so they never enter the processor.
|
// configured targetGenders here so they never enter the processor.
|
||||||
|
|||||||
@@ -105,6 +105,31 @@ describe('MonitorService', () => {
|
|||||||
expect(messagesRepo.hasSentTo).toHaveBeenCalledWith('already_msgd');
|
expect(messagesRepo.hasSentTo).toHaveBeenCalledWith('already_msgd');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('records seller posts as processed and continues to a clean candidate', async () => {
|
||||||
|
const sellerPost: RedditPost = {
|
||||||
|
id: 't3_seller', subreddit: 'sub1', author: 'seller', title: 'My page',
|
||||||
|
selftext: 'OnlyFans in bio', url: 'http://example.com/1.jpg', permalink: '/r/sub1/1', thumbnail: '',
|
||||||
|
is_self: false, created_utc: 2, over_18: true, spoiler: false,
|
||||||
|
};
|
||||||
|
const cleanPost: RedditPost = {
|
||||||
|
id: 't3_clean', subreddit: 'sub1', author: 'clean', title: 'Hello',
|
||||||
|
url: 'http://example.com/2.jpg', permalink: '/r/sub1/2', thumbnail: '',
|
||||||
|
is_self: false, created_utc: 1, over_18: true, spoiler: false,
|
||||||
|
};
|
||||||
|
const redditApi = createMockRedditApi([sellerPost, cleanPost]);
|
||||||
|
vi.mocked(redditApi.isSellerProfile).mockImplementation(async (username) => username === 'seller');
|
||||||
|
const postsRepo = createMockPostsRepo();
|
||||||
|
const config = getTestConfig({ reddit: { subreddits: [{ name: 'sub1', type: 'image' }] } });
|
||||||
|
monitor = new MonitorService(redditApi, postsRepo, createMockMessagesRepo(), config, logger, 0);
|
||||||
|
|
||||||
|
await expect(monitor.getUnprocessedPosts()).resolves.toEqual([cleanPost]);
|
||||||
|
expect(redditApi.isSellerProfile).toHaveBeenCalledWith(
|
||||||
|
'seller', config.reddit.sellerKeywords, config.reddit.sellerDomains, sellerPost,
|
||||||
|
);
|
||||||
|
expect(postsRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ id: 't3_seller' }));
|
||||||
|
expect(postsRepo.setError).toHaveBeenCalledWith('t3_seller', 'Seller signals detected');
|
||||||
|
});
|
||||||
|
|
||||||
it('returns newest post when multiple valid posts exist', async () => {
|
it('returns newest post when multiple valid posts exist', async () => {
|
||||||
const post1: RedditPost = {
|
const post1: RedditPost = {
|
||||||
id: 't3_1', subreddit: 'sub1', author: 'user1', title: 'Older',
|
id: 't3_1', subreddit: 'sub1', author: 'user1', title: 'Older',
|
||||||
|
|||||||
@@ -242,6 +242,38 @@ describe('RedditApiClient', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('isSellerProfile', () => {
|
describe('isSellerProfile', () => {
|
||||||
|
it('returns true for a seller signal in the current post without fetching the profile', async () => {
|
||||||
|
const get = vi.fn();
|
||||||
|
vi.mocked(axios.create).mockReturnValue({ get } as any);
|
||||||
|
const client2 = new RedditApiClient(getTestConfig(), logger);
|
||||||
|
const post = {
|
||||||
|
id: 'post-1', name: 't3_post-1', subreddit: 'test', author: 'testuser',
|
||||||
|
title: 'Come and subscribe', selftext: 'My page is at https://onlyfans.com/example',
|
||||||
|
url: 'https://reddit.com/post-1', permalink: '/r/test/post-1', thumbnail: '',
|
||||||
|
is_self: true, created_utc: 1, over_18: true, spoiler: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
await expect(client2.isSellerProfile('testuser', ['onlyfans'], ['onlyfans.com'], post))
|
||||||
|
.resolves.toBe(true);
|
||||||
|
expect(get).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('detects seller signals in recent profile activity when the bio is clean', async () => {
|
||||||
|
const get = vi.fn()
|
||||||
|
.mockResolvedValueOnce({ data: { data: { subreddit: { public_description: 'Just me' } } } })
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
data: { data: { children: [{ data: { title: 'New content', subreddit: 'OnlyFansPromotions' } }] } },
|
||||||
|
});
|
||||||
|
vi.mocked(axios.create).mockReturnValue({ get } as any);
|
||||||
|
const client2 = new RedditApiClient(getTestConfig(), logger);
|
||||||
|
|
||||||
|
await expect(client2.isSellerProfile('testuser', ['onlyfans'])).resolves.toBe(true);
|
||||||
|
expect(get).toHaveBeenNthCalledWith(2, '/user/testuser/overview.json', {
|
||||||
|
params: { limit: 25, raw_json: 1 },
|
||||||
|
headers: {},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('returns true when seller keyword matches', async () => {
|
it('returns true when seller keyword matches', async () => {
|
||||||
const mockResponse = {
|
const mockResponse = {
|
||||||
data: {
|
data: {
|
||||||
|
|||||||
Reference in new issue
Block a user