krrishg/indie
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prune old sites

Browse Source
This commit is contained in:
Krrish Ghimire
2026-07-11 01:28:09 +05:45
parent e684154938
commit c362219a40
8 changed files with 485 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/src/main/java/com/krrishg/controller/LLMController.java
View File

@@ -162,4 +162,17 @@ public class LLMController {
return ResponseEntity.notFound().build();
}
}
@PostMapping("/versions/{siteId}/prune")
public ResponseEntity<?> pruneVersions(
@AuthenticationPrincipal UserPrincipal principal,
@PathVariable UUID siteId,
@RequestBody(required = false) Map<String, Object> request) {
int keep = 10;
if (request != null && request.containsKey("keep")) {
keep = ((Number) request.get("keep")).intValue();
}
int deleted = llmService.pruneVersions(siteId, keep);
return ResponseEntity.ok(Map.of("deleted", deleted));
}
}

1
backend/src/main/java/com/krrishg/dto/SiteStructure.java
View File

@@ -35,5 +35,6 @@ public class SiteStructure {
private String seoDescription;
private int orderIndex;
private String rawHtml;
private boolean deleted;
}
}

25
backend/src/main/java/com/krrishg/service/LLMResponseParser.java
View File

@@ -31,7 +31,17 @@ public class LLMResponseParser {
String json = extractJson(llmOutput);
try {
JsonNode root = objectMapper.readTree(json);
return parseSiteStructure(root);
return parseSiteStructure(root, true);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("Failed to parse LLM output as JSON: " + e.getMessage());
}
}
public SiteStructure parseLenient(String llmOutput) {
String json = extractJson(llmOutput);
try {
JsonNode root = objectMapper.readTree(json);
return parseSiteStructure(root, false);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException("Failed to parse LLM output as JSON: " + e.getMessage());
}
@@ -69,7 +79,7 @@ public class LLMResponseParser {
return trimmed;
}
private SiteStructure parseSiteStructure(JsonNode root) {
private SiteStructure parseSiteStructure(JsonNode root, boolean requirePages) {
SiteStructure.SiteStructureBuilder builder = SiteStructure.builder();
if (root.has("theme")) {
@@ -79,13 +89,13 @@ public class LLMResponseParser {
List<SiteStructure.PageStructure> pages = new ArrayList<>();
if (root.has("pages") && root.get("pages").isArray()) {
for (JsonNode pageNode : root.get("pages")) {
pages.add(parsePage(pageNode));
pages.add(parsePage(pageNode, !requirePages));
}
}
builder.pages(pages);
SiteStructure structure = builder.build();
if (structure.getPages() == null || structure.getPages().isEmpty()) {
if (requirePages && (structure.getPages() == null || structure.getPages().isEmpty())) {
throw new IllegalArgumentException("LLM output must contain at least one page");
}
@@ -118,7 +128,7 @@ public class LLMResponseParser {
return map;
}
private SiteStructure.PageStructure parsePage(JsonNode pageNode) {
private SiteStructure.PageStructure parsePage(JsonNode pageNode, boolean lenient) {
SiteStructure.PageStructure.PageStructureBuilder builder = SiteStructure.PageStructure.builder();
if (pageNode.has("id")) {
@@ -145,9 +155,12 @@ public class LLMResponseParser {
if (pageNode.has("rawHtml")) {
builder.rawHtml(pageNode.get("rawHtml").asText());
}
if (pageNode.has("deleted")) {
builder.deleted(pageNode.get("deleted").asBoolean());
}
SiteStructure.PageStructure page = builder.build();
if (page.getTitle() == null || page.getSlug() == null) {
if (!lenient && !page.isDeleted() && (page.getTitle() == null || page.getSlug() == null)) {
throw new IllegalArgumentException("Page must have title and slug");
}

155
backend/src/main/java/com/krrishg/service/LLMService.java
View File

@@ -29,8 +29,12 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -38,7 +42,6 @@ import java.util.stream.Collectors;
public class LLMService {
private static final Logger log = LoggerFactory.getLogger(LLMService.class);
private static final int MAX_VERSIONS_PER_SITE = 100;
private final ProviderFactory providerFactory;
private final LLMResponseParser responseParser;
@@ -48,6 +51,8 @@ public class LLMService {
private final EncryptionService encryptionService;
private final String systemPrompt;
private final String refinerPrompt;
private final int maxVersionsPerSite;
private final double refineTemperature;
public LLMService(ProviderFactory providerFactory,
LLMResponseParser responseParser,
@@ -57,7 +62,9 @@ public class LLMService {
EncryptionService encryptionService,
ResourceLoader resourceLoader,
@Value("${indie.llm.system-prompt-path:classpath:prompts/site-generator.txt}") String promptPath,
@Value("${indie.llm.refiner-prompt-path:classpath:prompts/site-refiner.txt}") String refinerPath) {
@Value("${indie.llm.refiner-prompt-path:classpath:prompts/site-refiner.txt}") String refinerPath,
@Value("${indie.llm.max-versions-per-site:10}") int maxVersionsPerSite,
@Value("${indie.llm.refine.temperature:0.3}") double refineTemperature) {
this.providerFactory = providerFactory;
this.responseParser = responseParser;
this.referenceService = referenceService;
@@ -66,6 +73,8 @@ public class LLMService {
this.encryptionService = encryptionService;
this.systemPrompt = loadSystemPrompt(resourceLoader, promptPath);
this.refinerPrompt = loadSystemPrompt(resourceLoader, refinerPath);
this.maxVersionsPerSite = maxVersionsPerSite;
this.refineTemperature = refineTemperature;
}
public SiteStructure generateSite(UUID userId, UUID siteId, String prompt,
@@ -121,29 +130,54 @@ public class LLMService {
String combinedPrompt = "Current site structure:\n" + currentJson
+ "\n\nRefinement request: " + refinementPrompt
+ referenceContext
+ "\n\nOutput the complete updated JSON structure with the requested changes applied.";
+ "\n\nOutput only the fields that changed. Fields you omit will remain unchanged."
+ " Include the \"id\" field for any page you modify or add."
+ " To delete a page, include it with \"id\" and \"deleted\": true."
+ " Pages not included in your output will stay as-is.";
LLMOptions options = LLMOptions.builder()
.temperature(0.5)
.temperature(refineTemperature)
.build();
LLMResult result = callLLM(provider, resolvedKey, resolvedBaseUrl, resolvedModel, refinerPrompt, combinedPrompt, references, options);
log.info("LLM refinement completed: model={}, latency={}ms, tokens={}+{}",
result.getModel(), result.getLatencyMs(),
result.getPromptTokens(), result.getCompletionTokens());
SiteStructure structure = responseParser.parse(result.getContent());
saveVersion(userId, siteId, refinementPrompt, references, result.getContent());
SiteStructure patch = responseParser.parseLenient(result.getContent());
SiteStructure merged = mergePatches(currentStructure, patch);
saveVersion(userId, siteId, refinementPrompt, references, serializeStructure(merged));
return structure;
return merged;
}
public List<GenerationVersion> getVersions(UUID siteId) {
Query query = new Query(Criteria.where("siteId").is(siteId))
.with(Sort.by(Sort.Direction.DESC, "versionNumber"))
.limit(MAX_VERSIONS_PER_SITE);
.limit(maxVersionsPerSite);
return mongoTemplate.find(query, GenerationVersion.class);
}
public int pruneVersions(UUID siteId) {
return pruneVersions(siteId, maxVersionsPerSite);
}
public int pruneVersions(UUID siteId, int keep) {
Query countQuery = new Query(Criteria.where("siteId").is(siteId));
long total = mongoTemplate.count(countQuery, GenerationVersion.class);
if (total <= keep) {
return 0;
}
long toDelete = total - keep;
Query findQuery = new Query(Criteria.where("siteId").is(siteId))
.with(Sort.by(Sort.Direction.ASC, "versionNumber"))
.limit((int) toDelete);
List<GenerationVersion> oldVersions = mongoTemplate.find(findQuery, GenerationVersion.class);
for (GenerationVersion v : oldVersions) {
mongoTemplate.remove(v);
}
return oldVersions.size();
}
public SiteStructure restoreVersion(String versionId) {
GenerationVersion version = mongoTemplate.findById(versionId, GenerationVersion.class);
if (version == null) {
@@ -160,6 +194,110 @@ public class LLMService {
mongoTemplate.remove(version);
}
private SiteStructure mergePatches(SiteStructure current, SiteStructure patch) {
SiteStructure.SiteStructureBuilder builder = SiteStructure.builder();
if (patch.getTheme() != null) {
builder.theme(mergeTheme(current.getTheme(), patch.getTheme()));
} else {
builder.theme(current.getTheme());
}
if (patch.getPages() != null && !patch.getPages().isEmpty()) {
Map<String, SiteStructure.PageStructure> currentPageMap = new HashMap<>();
for (SiteStructure.PageStructure page : current.getPages()) {
if (page.getId() != null) {
currentPageMap.put(page.getId(), page);
}
}
Set<String> deletedIds = new HashSet<>();
for (SiteStructure.PageStructure patchPage : patch.getPages()) {
if (patchPage.isDeleted() && patchPage.getId() != null) {
deletedIds.add(patchPage.getId());
}
}
List<SiteStructure.PageStructure> mergedPages = new ArrayList<>();
for (SiteStructure.PageStructure currentPage : current.getPages()) {
if (currentPage.getId() != null && deletedIds.contains(currentPage.getId())) {
continue;
}
SiteStructure.PageStructure matchingPatch = null;
if (currentPage.getId() != null) {
for (SiteStructure.PageStructure pp : patch.getPages()) {
if (currentPage.getId().equals(pp.getId())) {
matchingPatch = pp;
break;
}
}
}
if (matchingPatch != null && !matchingPatch.isDeleted()) {
mergedPages.add(mergePage(currentPage, matchingPatch));
} else {
mergedPages.add(currentPage);
}
}
for (SiteStructure.PageStructure patchPage : patch.getPages()) {
if (patchPage.isDeleted()) continue;
String pid = patchPage.getId();
if (pid == null || !currentPageMap.containsKey(pid)) {
mergedPages.add(patchPage);
}
}
builder.pages(mergedPages);
} else {
builder.pages(current.getPages());
}
return builder.build();
}
private SiteStructure.ThemeConfig mergeTheme(SiteStructure.ThemeConfig current, SiteStructure.ThemeConfig patch) {
SiteStructure.ThemeConfig.ThemeConfigBuilder builder = SiteStructure.ThemeConfig.builder();
if (patch.getColors() != null) {
Map<String, String> mergedColors = new HashMap<>(current.getColors() != null ? current.getColors() : Map.of());
mergedColors.putAll(patch.getColors());
builder.colors(mergedColors);
} else {
builder.colors(current.getColors());
}
if (patch.getFonts() != null) {
Map<String, String> mergedFonts = new HashMap<>(current.getFonts() != null ? current.getFonts() : Map.of());
mergedFonts.putAll(patch.getFonts());
builder.fonts(mergedFonts);
} else {
builder.fonts(current.getFonts());
}
if (patch.getSpacing() != null) {
Map<String, String> mergedSpacing = new HashMap<>(current.getSpacing() != null ? current.getSpacing() : Map.of());
mergedSpacing.putAll(patch.getSpacing());
builder.spacing(mergedSpacing);
} else {
builder.spacing(current.getSpacing());
}
return builder.build();
}
private SiteStructure.PageStructure mergePage(SiteStructure.PageStructure current, SiteStructure.PageStructure patch) {
SiteStructure.PageStructure.PageStructureBuilder builder = SiteStructure.PageStructure.builder();
builder.id(patch.getId() != null ? patch.getId() : current.getId());
builder.title(patch.getTitle() != null ? patch.getTitle() : current.getTitle());
builder.slug(patch.getSlug() != null ? patch.getSlug() : current.getSlug());
builder.seoTitle(patch.getSeoTitle() != null ? patch.getSeoTitle() : current.getSeoTitle());
builder.seoDescription(patch.getSeoDescription() != null ? patch.getSeoDescription() : current.getSeoDescription());
builder.orderIndex(patch.getOrderIndex() != 0 ? patch.getOrderIndex() : current.getOrderIndex());
builder.rawHtml(patch.getRawHtml() != null ? patch.getRawHtml() : current.getRawHtml());
builder.deleted(patch.isDeleted());
return builder.build();
}
private String resolveApiKey(UUID userId, String provider, String apiKey) {
if (apiKey != null && !apiKey.isBlank()) {
return apiKey;
@@ -247,6 +385,7 @@ public class LLMService {
.createdAt(LocalDateTime.now())
.build();
mongoTemplate.save(version);
pruneVersions(siteId);
}
private int getNextVersionNumber(UUID siteId) {

4
backend/src/main/resources/application.yml
View File

@@ -73,6 +73,10 @@ indie:
llm:
read-timeout: ${INDIE_LLM_READ_TIMEOUT:180s}
max-versions-per-site: ${INDIE_LLM_MAX_VERSIONS:10}
refine:
temperature: ${INDIE_LLM_REFINE_TEMPERATURE:0.3}
read-timeout: ${INDIE_LLM_REFINE_READ_TIMEOUT:300s}
rate-limit:
max-requests-per-hour: ${INDIE_LLM_RATE_LIMIT:10}
providers:

195
backend/src/test/java/com/krrishg/service/LLMServiceTest.java
View File

@@ -82,7 +82,8 @@ class LLMServiceTest {
llmService = new LLMService(providerFactory, responseParser, referenceService,
mongoTemplate, userRepository, encryptionService,
resourceLoader, "classpath:prompts/site-generator.txt",
"classpath:prompts/site-refiner.txt");
"classpath:prompts/site-refiner.txt",
10, 0.3);
}
@Test
@@ -210,9 +211,11 @@ class LLMServiceTest {
@Test
void refineSiteReturnsRefinedStructure() {
String pageId = UUID.randomUUID().toString();
SiteStructure current = SiteStructure.builder()
.pages(List.of(
SiteStructure.PageStructure.builder()
.id(pageId)
.title("Old")
.slug("old")
.rawHtml("<p>Old content</p>")
@@ -223,16 +226,202 @@ class LLMServiceTest {
geminiProvider.withContent("""
{
"pages": [
{"title": "Refined", "slug": "refined", "rawHtml": "<p>Hero</p>"}
{"id": "__PAGE_ID__", "title": "Refined", "slug": "refined", "rawHtml": "<p>Updated</p>"}
]
}
""");
""".replace("__PAGE_ID__", pageId));
when(mongoTemplate.findOne(any(Query.class), eq(GenerationVersion.class))).thenReturn(null);
SiteStructure result = llmService.refineSite(userId, siteId, current, "Make it better", null, "gemini", null);
assertEquals("Refined", result.getPages().get(0).getTitle());
assertEquals("<p>Updated</p>", result.getPages().get(0).getRawHtml());
verify(mongoTemplate).save(any(GenerationVersion.class));
}
@Test
void refineSitePreservesUnchangedPages() {
String pageId1 = UUID.randomUUID().toString();
String pageId2 = UUID.randomUUID().toString();
SiteStructure current = SiteStructure.builder()
.theme(SiteStructure.ThemeConfig.builder()
.colors(Map.of("primary", "#000", "background", "#fff"))
.build())
.pages(List.of(
SiteStructure.PageStructure.builder()
.id(pageId1)
.title("Home")
.slug("home")
.rawHtml("<p>Home content</p>")
.build(),
SiteStructure.PageStructure.builder()
.id(pageId2)
.title("About")
.slug("about")
.rawHtml("<p>About content</p>")
.build()
))
.build();
geminiProvider.withContent("""
{
"theme": {
"colors": {
"primary": "#1a1a2e",
"accent": "#e94560"
}
},
"pages": [
{"id": "__PAGE_ID__", "rawHtml": "<p>Updated home</p>"}
]
}
""".replace("__PAGE_ID__", pageId1));
when(mongoTemplate.findOne(any(Query.class), eq(GenerationVersion.class))).thenReturn(null);
SiteStructure result = llmService.refineSite(userId, siteId, current, "Change primary color", null, "gemini", null);
assertEquals("#1a1a2e", result.getTheme().getColors().get("primary"));
assertEquals("#fff", result.getTheme().getColors().get("background"));
assertNotNull(result.getTheme().getColors().get("accent"));
assertEquals(2, result.getPages().size());
assertEquals("<p>Updated home</p>", result.getPages().get(0).getRawHtml());
assertEquals("About", result.getPages().get(1).getTitle());
assertEquals("<p>About content</p>", result.getPages().get(1).getRawHtml());
verify(mongoTemplate).save(any(GenerationVersion.class));
}
@Test
void refineSiteDeletesPage() {
String pageId1 = UUID.randomUUID().toString();
String pageId2 = UUID.randomUUID().toString();
SiteStructure current = SiteStructure.builder()
.pages(List.of(
SiteStructure.PageStructure.builder()
.id(pageId1)
.title("Home")
.slug("home")
.rawHtml("<p>Home</p>")
.build(),
SiteStructure.PageStructure.builder()
.id(pageId2)
.title("About")
.slug("about")
.rawHtml("<p>About</p>")
.build()
))
.build();
geminiProvider.withContent("""
{
"pages": [
{"id": "__PAGE_ID__", "deleted": true}
]
}
""".replace("__PAGE_ID__", pageId1));
when(mongoTemplate.findOne(any(Query.class), eq(GenerationVersion.class))).thenReturn(null);
SiteStructure result = llmService.refineSite(userId, siteId, current, "Remove Home page", null, "gemini", null);
assertEquals(1, result.getPages().size());
assertEquals("About", result.getPages().get(0).getTitle());
verify(mongoTemplate).save(any(GenerationVersion.class));
}
@Test
void refineSiteAddsNewPage() {
String existingId = UUID.randomUUID().toString();
SiteStructure current = SiteStructure.builder()
.pages(List.of(
SiteStructure.PageStructure.builder()
.id(existingId)
.title("Home")
.slug("home")
.rawHtml("<p>Home</p>")
.build()
))
.build();
geminiProvider.withContent("""
{
"pages": [
{"title": "Contact", "slug": "contact", "rawHtml": "<p>Contact form</p>"}
]
}
""");
when(mongoTemplate.findOne(any(Query.class), eq(GenerationVersion.class))).thenReturn(null);
SiteStructure result = llmService.refineSite(userId, siteId, current, "Add Contact page", null, "gemini", null);
assertEquals(2, result.getPages().size());
assertEquals("Contact", result.getPages().get(1).getTitle());
verify(mongoTemplate).save(any(GenerationVersion.class));
}
@Test
void pruneVersionsDeletesOldestVersions() {
lenient().when(mongoTemplate.count(any(Query.class), eq(GenerationVersion.class))).thenReturn(15L);
List<GenerationVersion> oldVersions = List.of(
GenerationVersion.builder().id("v1").siteId(siteId).versionNumber(1).build(),
GenerationVersion.builder().id("v2").siteId(siteId).versionNumber(2).build(),
GenerationVersion.builder().id("v3").siteId(siteId).versionNumber(3).build(),
GenerationVersion.builder().id("v4").siteId(siteId).versionNumber(4).build(),
GenerationVersion.builder().id("v5").siteId(siteId).versionNumber(5).build()
);
lenient().when(mongoTemplate.find(any(Query.class), eq(GenerationVersion.class))).thenReturn(oldVersions);
int deleted = llmService.pruneVersions(siteId, 10);
assertEquals(5, deleted);
}
@Test
void pruneVersionsDoesNothingWhenUnderLimit() {
lenient().when(mongoTemplate.count(any(Query.class), eq(GenerationVersion.class))).thenReturn(8L);
int deleted = llmService.pruneVersions(siteId, 10);
assertEquals(0, deleted);
}
@Test
void saveVersionTriggersAutoCleanup() {
String pageId = UUID.randomUUID().toString();
when(mongoTemplate.findOne(any(Query.class), eq(GenerationVersion.class))).thenReturn(null);
lenient().when(mongoTemplate.count(any(Query.class), eq(GenerationVersion.class))).thenReturn(15L);
lenient().when(mongoTemplate.find(any(Query.class), eq(GenerationVersion.class))).thenReturn(List.of(
GenerationVersion.builder().id("old").siteId(siteId).versionNumber(1).build(),
GenerationVersion.builder().id("older").siteId(siteId).versionNumber(2).build(),
GenerationVersion.builder().id("oldest").siteId(siteId).versionNumber(3).build(),
GenerationVersion.builder().id("ancient").siteId(siteId).versionNumber(4).build(),
GenerationVersion.builder().id("prehistoric").siteId(siteId).versionNumber(5).build()
));
geminiProvider.withContent("""
{
"pages": [
{"id": "__PAGE_ID__", "title": "Home", "slug": "home", "rawHtml": "<p>Test</p>"}
]
}
""".replace("__PAGE_ID__", pageId));
SiteStructure current = SiteStructure.builder()
.pages(List.of(
SiteStructure.PageStructure.builder()
.id(pageId)
.title("Home")
.slug("home")
.rawHtml("<p>Old</p>")
.build()
))
.build();
llmService.refineSite(userId, siteId, current, "Update", null, "gemini", null);
verify(mongoTemplate).save(any(GenerationVersion.class));
}