Improve seller account filtering
This commit is contained in:
1 parent
d819a0f1e6
commit
f4e8fdc42f
5 files changed
+178
-52
No files matched your search
@@ -254,29 +254,6 @@ export async function runMainLoop(app: AppContainer): Promise<void> {
|
||||
processedAt: Date.now(),
|
||||
});
|
||||
|
||||
// Skip OF/seller accounts before spending on AI.
|
||||
if (
|
||||
config.reddit.filterSellers &&
|
||||
(config.reddit.sellerKeywords.length > 0 || config.reddit.sellerDomains.length > 0)
|
||||
) {
|
||||
const isSeller = await redditApi.isSellerProfile(
|
||||
post.author,
|
||||
config.reddit.sellerKeywords,
|
||||
config.reddit.sellerDomains
|
||||
);
|
||||
if (isSeller) {
|
||||
logger.info(`Skipping u/${post.author} - seller/OnlyFans profile detected`);
|
||||
webServer?.logActivity(
|
||||
'skipped',
|
||||
`Skipped post by u/${post.author} (seller/OnlyFans profile)`,
|
||||
post.author,
|
||||
);
|
||||
webServer?.pushLog('info', `🚫 Skipped u/${post.author} - seller/OnlyFans profile`);
|
||||
postsRepo.setError(post.id, 'Seller profile detected');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh examples weighted toward this post's subreddit.
|
||||
const subredditExamples = responseChecker.getSuccessfulExamples(3, post.subreddit);
|
||||
processorService.setSuccessfulExamples(subredditExamples);
|
||||
|
||||
+95
-29
@@ -48,6 +48,38 @@ export interface RedditPost {
|
||||
link_flair_text?: string | null;
|
||||
}
|
||||
|
||||
interface RedditListingChild {
|
||||
data?: {
|
||||
title?: unknown;
|
||||
selftext?: unknown;
|
||||
body?: unknown;
|
||||
url?: unknown;
|
||||
subreddit?: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
function joinText(values: unknown[]): string {
|
||||
return values.filter((value): value is string => typeof value === 'string').join(' ');
|
||||
}
|
||||
|
||||
/** Match configured seller signals in user-controlled text. */
|
||||
export function findSellerSignal(
|
||||
text: string,
|
||||
keywords: string[],
|
||||
domains: string[],
|
||||
): { kind: 'keyword' | 'domain'; value: string } | null {
|
||||
const normalized = text.toLocaleLowerCase();
|
||||
for (const keyword of keywords) {
|
||||
const candidate = keyword.trim().toLocaleLowerCase();
|
||||
if (candidate && normalized.includes(candidate)) {
|
||||
return { kind: 'keyword', value: keyword };
|
||||
}
|
||||
}
|
||||
|
||||
const domain = findSellerDomain(text, domains);
|
||||
return domain ? { kind: 'domain', value: domain } : null;
|
||||
}
|
||||
|
||||
// Extracts URL-ish tokens (with or without scheme) and returns the first
|
||||
// domain match. Matches `host` or any subdomain (`*.host`), case-insensitive.
|
||||
export function findSellerDomain(text: string, domains: string[]): string | null {
|
||||
@@ -249,10 +281,28 @@ export class RedditApiClient {
|
||||
async isSellerProfile(
|
||||
username: string,
|
||||
sellerKeywords: string[],
|
||||
sellerDomains: string[] = []
|
||||
sellerDomains: string[] = [],
|
||||
currentPost?: RedditPost,
|
||||
): Promise<boolean> {
|
||||
if (sellerKeywords.length === 0 && sellerDomains.length === 0) return false;
|
||||
|
||||
// Check the post first. This catches direct promotional posts without an
|
||||
// extra request and also covers accounts whose profile bio is deliberately
|
||||
// vague.
|
||||
if (currentPost) {
|
||||
const postSignal = findSellerSignal(
|
||||
joinText([currentPost.title, currentPost.selftext, currentPost.url]),
|
||||
sellerKeywords,
|
||||
sellerDomains,
|
||||
);
|
||||
if (postSignal) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller from current post (matched ${postSignal.kind}: "${postSignal.value}")`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const cookieHeader = await this.getCookieHeader();
|
||||
const headers: Record<string, string> = {};
|
||||
@@ -260,51 +310,67 @@ export class RedditApiClient {
|
||||
headers['Cookie'] = cookieHeader;
|
||||
}
|
||||
|
||||
const response = await this.client.get(`/user/${username}/about.json`, {
|
||||
const response = await this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
|
||||
params: { raw_json: 1 },
|
||||
headers,
|
||||
});
|
||||
|
||||
const userData = response.data?.data;
|
||||
if (!userData) return false;
|
||||
|
||||
// Combine all text fields to scan
|
||||
const textToCheck = [
|
||||
userData.subreddit?.public_description || '',
|
||||
userData.subreddit?.title || '',
|
||||
userData.subreddit?.display_name_prefixed || '',
|
||||
]
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
|
||||
for (const keyword of sellerKeywords) {
|
||||
if (textToCheck.includes(keyword.toLowerCase())) {
|
||||
if (userData) {
|
||||
const profileText = joinText([
|
||||
userData.subreddit?.public_description || '',
|
||||
userData.subreddit?.title || '',
|
||||
userData.subreddit?.display_name_prefixed || '',
|
||||
]);
|
||||
const profileSignal = findSellerSignal(profileText, sellerKeywords, sellerDomains);
|
||||
if (profileSignal) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller profile (matched keyword: "${keyword}")`
|
||||
`u/${username} flagged as seller profile (matched ${profileSignal.kind}: "${profileSignal.value}")`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (sellerDomains.length > 0) {
|
||||
const matchedDomain = findSellerDomain(textToCheck, sellerDomains);
|
||||
if (matchedDomain) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller profile (matched domain: "${matchedDomain}")`
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error) && error.response?.status === 404) {
|
||||
this.logger.warn(`User u/${username} profile not found`);
|
||||
return false;
|
||||
}
|
||||
this.logger.warn(`Failed to check profile for u/${username}:`, error);
|
||||
return false; // Don't block on errors - let it through
|
||||
}
|
||||
|
||||
// Bios are often clean while recent posts advertise the paid account.
|
||||
// Scan a small overview page, including comment bodies and subreddit
|
||||
// names, before allowing the candidate through.
|
||||
try {
|
||||
const cookieHeader = await this.getCookieHeader();
|
||||
const headers: Record<string, string> = {};
|
||||
if (cookieHeader) headers.Cookie = cookieHeader;
|
||||
const response = await this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
|
||||
params: { limit: 25, raw_json: 1 },
|
||||
headers,
|
||||
});
|
||||
const children: RedditListingChild[] = Array.isArray(response.data?.data?.children)
|
||||
? response.data.data.children
|
||||
: [];
|
||||
const activityText = children.map(({ data }) => joinText([
|
||||
data?.title,
|
||||
data?.selftext,
|
||||
data?.body,
|
||||
data?.url,
|
||||
data?.subreddit,
|
||||
])).join(' ');
|
||||
const activitySignal = findSellerSignal(activityText, sellerKeywords, sellerDomains);
|
||||
if (activitySignal) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller from recent activity (matched ${activitySignal.kind}: "${activitySignal.value}")`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to check recent activity for u/${username}:`, error);
|
||||
}
|
||||
|
||||
return false; // Fail open if Reddit profile endpoints are unavailable.
|
||||
}
|
||||
|
||||
getPostUrl(post: RedditPost): string {
|
||||
|
||||
@@ -48,6 +48,32 @@ export class MonitorService {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
this.config.reddit.filterSellers &&
|
||||
(this.config.reddit.sellerKeywords.length > 0 || this.config.reddit.sellerDomains.length > 0)
|
||||
) {
|
||||
const isSeller = await this.redditApi.isSellerProfile(
|
||||
post.author,
|
||||
this.config.reddit.sellerKeywords,
|
||||
this.config.reddit.sellerDomains,
|
||||
post,
|
||||
);
|
||||
if (isSeller) {
|
||||
this.logger.info(`Skipping post ${post.id} from u/${post.author} - seller signals detected`);
|
||||
this.postsRepo.insert({
|
||||
id: post.id,
|
||||
subreddit: post.subreddit,
|
||||
author: post.author,
|
||||
title: post.title,
|
||||
postUrl: this.redditApi.getPostUrl(post),
|
||||
imageUrl: subreddit.type === 'image' ? this.redditApi.getImageUrl(post) : null,
|
||||
processedAt: Date.now(),
|
||||
});
|
||||
this.postsRepo.setError(post.id, 'Seller signals detected');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Chat subs almost always have a self-declared gender tag in the
|
||||
// title/body (e.g. [25F], M4F). Drop posts that don't match the
|
||||
// configured targetGenders here so they never enter the processor.
|
||||
|
||||
@@ -105,6 +105,31 @@ describe('MonitorService', () => {
|
||||
expect(messagesRepo.hasSentTo).toHaveBeenCalledWith('already_msgd');
|
||||
});
|
||||
|
||||
it('records seller posts as processed and continues to a clean candidate', async () => {
|
||||
const sellerPost: RedditPost = {
|
||||
id: 't3_seller', subreddit: 'sub1', author: 'seller', title: 'My page',
|
||||
selftext: 'OnlyFans in bio', url: 'http://example.com/1.jpg', permalink: '/r/sub1/1', thumbnail: '',
|
||||
is_self: false, created_utc: 2, over_18: true, spoiler: false,
|
||||
};
|
||||
const cleanPost: RedditPost = {
|
||||
id: 't3_clean', subreddit: 'sub1', author: 'clean', title: 'Hello',
|
||||
url: 'http://example.com/2.jpg', permalink: '/r/sub1/2', thumbnail: '',
|
||||
is_self: false, created_utc: 1, over_18: true, spoiler: false,
|
||||
};
|
||||
const redditApi = createMockRedditApi([sellerPost, cleanPost]);
|
||||
vi.mocked(redditApi.isSellerProfile).mockImplementation(async (username) => username === 'seller');
|
||||
const postsRepo = createMockPostsRepo();
|
||||
const config = getTestConfig({ reddit: { subreddits: [{ name: 'sub1', type: 'image' }] } });
|
||||
monitor = new MonitorService(redditApi, postsRepo, createMockMessagesRepo(), config, logger, 0);
|
||||
|
||||
await expect(monitor.getUnprocessedPosts()).resolves.toEqual([cleanPost]);
|
||||
expect(redditApi.isSellerProfile).toHaveBeenCalledWith(
|
||||
'seller', config.reddit.sellerKeywords, config.reddit.sellerDomains, sellerPost,
|
||||
);
|
||||
expect(postsRepo.insert).toHaveBeenCalledWith(expect.objectContaining({ id: 't3_seller' }));
|
||||
expect(postsRepo.setError).toHaveBeenCalledWith('t3_seller', 'Seller signals detected');
|
||||
});
|
||||
|
||||
it('returns newest post when multiple valid posts exist', async () => {
|
||||
const post1: RedditPost = {
|
||||
id: 't3_1', subreddit: 'sub1', author: 'user1', title: 'Older',
|
||||
|
||||
@@ -242,6 +242,38 @@ describe('RedditApiClient', () => {
|
||||
});
|
||||
|
||||
describe('isSellerProfile', () => {
|
||||
it('returns true for a seller signal in the current post without fetching the profile', async () => {
|
||||
const get = vi.fn();
|
||||
vi.mocked(axios.create).mockReturnValue({ get } as any);
|
||||
const client2 = new RedditApiClient(getTestConfig(), logger);
|
||||
const post = {
|
||||
id: 'post-1', name: 't3_post-1', subreddit: 'test', author: 'testuser',
|
||||
title: 'Come and subscribe', selftext: 'My page is at https://onlyfans.com/example',
|
||||
url: 'https://reddit.com/post-1', permalink: '/r/test/post-1', thumbnail: '',
|
||||
is_self: true, created_utc: 1, over_18: true, spoiler: false,
|
||||
};
|
||||
|
||||
await expect(client2.isSellerProfile('testuser', ['onlyfans'], ['onlyfans.com'], post))
|
||||
.resolves.toBe(true);
|
||||
expect(get).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('detects seller signals in recent profile activity when the bio is clean', async () => {
|
||||
const get = vi.fn()
|
||||
.mockResolvedValueOnce({ data: { data: { subreddit: { public_description: 'Just me' } } } })
|
||||
.mockResolvedValueOnce({
|
||||
data: { data: { children: [{ data: { title: 'New content', subreddit: 'OnlyFansPromotions' } }] } },
|
||||
});
|
||||
vi.mocked(axios.create).mockReturnValue({ get } as any);
|
||||
const client2 = new RedditApiClient(getTestConfig(), logger);
|
||||
|
||||
await expect(client2.isSellerProfile('testuser', ['onlyfans'])).resolves.toBe(true);
|
||||
expect(get).toHaveBeenNthCalledWith(2, '/user/testuser/overview.json', {
|
||||
params: { limit: 25, raw_json: 1 },
|
||||
headers: {},
|
||||
});
|
||||
});
|
||||
|
||||
it('returns true when seller keyword matches', async () => {
|
||||
const mockResponse = {
|
||||
data: {
|
||||
|
||||
Reference in new issue
Block a user