This commit is contained in:
2026-01-26 17:07:35 +01:00
parent ec82340f42
commit 6c185f2212
37 changed files with 3888 additions and 171 deletions
+4
View File
@@ -58,3 +58,7 @@ cmd/optimizer/optimizer
# Testing
.test/
testdata/
/data/optimizations/pending/5126_1769378005.json
/data/optimizations/pending/5620_1769378105.json
/data/optimizations/pending/6024_1769377905.json
/data/optimizations/pending/6198_1769379471.json
+1 -1
View File
@@ -22,7 +22,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o seo-optimizer ./cmd/op
FROM alpine:latest
# Install ca-certificates for HTTPS and tzdata for timezone support
RUN apk --no-cache add ca-certificates tzdata
RUN apk --no-cache add ca-certificates tzdata libavif libavif-tools
WORKDIR /app
+13 -1
View File
@@ -1,4 +1,4 @@
.PHONY: build run test clean docker-build docker-run docker-stop logs dev-setup lint fmt tidy
.PHONY: build run test clean docker-build docker-run docker-stop logs dev-setup lint fmt tidy docker-upgrade
# Build the application
build:
@@ -44,6 +44,17 @@ docker-stop:
@echo "Stopping docker-compose..."
docker-compose down
# Docker upgrade (rebuild + recreate with fresh images)
docker-upgrade:
@echo "Stopping docker-compose..."
docker-compose down --remove-orphans
@echo "Removing Docker images for seo-optimizer..."
@docker image rm -f seo-optimizer:latest 2>/dev/null || true
@echo "Pruning dangling images..."
@docker image prune -f
@echo "Rebuilding and starting with docker-compose..."
docker-compose up -d --build --force-recreate
# View logs
logs:
docker-compose logs -f
@@ -131,6 +142,7 @@ help:
@echo " docker-build - Build Docker image"
@echo " docker-run - Start with docker-compose"
@echo " docker-stop - Stop docker-compose"
@echo " docker-upgrade- Rebuild, prune images, and recreate containers"
@echo " logs - View docker-compose logs"
@echo " dev-setup - Set up development environment"
@echo " deps - Install dependencies"
+21 -1
View File
@@ -91,6 +91,8 @@ func main() {
fmt.Fprintln(os.Stderr, " newsletter [status]")
fmt.Fprintln(os.Stderr, " newsletter generate")
fmt.Fprintln(os.Stderr, " newsletter delete <id>")
fmt.Fprintln(os.Stderr, " orphans")
fmt.Fprintln(os.Stderr, " translate <post_id> [language]")
fmt.Fprintln(os.Stderr, " audit [limit]")
fmt.Fprintln(os.Stderr, " version")
fmt.Fprintln(os.Stderr, " shell")
@@ -339,6 +341,24 @@ func executeCommand(c *client, cmd string, args []string) error {
status := args[0]
handleSimpleGet(c, "/api/v1/newsletter?status="+status)
}
case "orphans":
handleSimpleGet(c, "/api/v1/orphans?source=en&target=es")
case "translate":
if len(args) < 1 {
return fmt.Errorf("translate requires <post_id>")
}
postID, err := parseIntArg("post_id", args[0])
if err != nil {
return err
}
targetLang := "es"
if len(args) > 1 {
targetLang = args[1]
}
handleSimplePost(c, "/api/v1/translate", map[string]interface{}{
"post_id": postID,
"target_lang": targetLang,
})
case "audit":
limit := 100
if len(args) > 0 {
@@ -372,7 +392,7 @@ func runInteractive(c *client) {
return
}
if line == "help" {
fmt.Println("Commands: health, version, pending [summary|text <draft_id>], optimize <id> [lang], status <job_id>, changes <post_id>, approve <draft_id>, reject <draft_id>, ideas [status|generate|draft <id>|publish <id>|delete <id>], outreach [status|generate|delete <id>], newsletter [status|generate|delete <id>], audit [limit], exit")
fmt.Println("Commands: health, version, pending [summary|text <draft_id>], optimize <id> [lang], status <job_id>, changes <post_id>, approve <draft_id>, reject <draft_id>, ideas [status|generate|draft <id>|publish <id>|delete <id>], outreach [status|generate|delete <id>], newsletter [status|generate|delete <id>], orphans, translate <post_id> [lang], audit [limit], exit")
continue
}
tokens := strings.Fields(line)
+88 -3
View File
@@ -6,18 +6,22 @@ import (
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
"seo-optimizer/internal/agent"
"seo-optimizer/internal/api"
"seo-optimizer/internal/brokenlinks"
"seo-optimizer/internal/config"
"seo-optimizer/internal/ideas"
"seo-optimizer/internal/imageaudit"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/newsletter"
"seo-optimizer/internal/scheduler"
"seo-optimizer/internal/seo"
"seo-optimizer/internal/storage"
"seo-optimizer/internal/translation"
"seo-optimizer/internal/wordpress"
)
@@ -91,6 +95,22 @@ func main() {
os.Exit(1)
}
// Initialize DeepSeek client for translations (optional)
var translationClient *agent.DeepSeekClient
if strings.TrimSpace(cfg.DeepSeek.APIKey) != "" {
translationClient = agent.NewDeepSeekClient(
cfg.DeepSeek.APIKey,
cfg.DeepSeek.BaseURL,
cfg.DeepSeek.Model,
cfg.DeepSeek.MaxTokens,
cfg.DeepSeek.Temperature,
log,
)
log.Info("DeepSeek translation client initialized", "model", cfg.DeepSeek.Model)
} else {
log.Warn("DeepSeek API key missing; translation feature will be unavailable")
}
// Initialize storage
store := storage.NewOptimizationStorage(cfg.Storage.BasePath, log)
if err := store.Initialize(); err != nil {
@@ -131,12 +151,43 @@ func main() {
}
log.Info("Newsletter storage initialized", "base_path", cfg.Newsletter.BasePath)
// Initialize broken link storage
linkStore := storage.NewBrokenLinkStorage(cfg.LinkChecker.BasePath, log)
if err := linkStore.Initialize(); err != nil {
log.Error("Failed to initialize broken link storage", "error", err)
os.Exit(1)
}
log.Info("Broken link storage initialized", "base_path", cfg.LinkChecker.BasePath)
// Initialize image audit storage
imageAuditStore := storage.NewImageAuditStorage(cfg.ImageAudit.BasePath, log)
if err := imageAuditStore.Initialize(); err != nil {
log.Error("Failed to initialize image audit storage", "error", err)
os.Exit(1)
}
log.Info("Image audit storage initialized", "base_path", cfg.ImageAudit.BasePath)
// Initialize idea service
ideaService := ideas.NewService(wpClient, llmClient, nil, ideaStore, cfg.Ideas, outreachStore, cfg.Outreach, cfg.Runtime.DryRun, log)
// Initialize newsletter service
newsService := newsletter.NewService(llmClient, nil, newsStore, cfg.Newsletter, log)
// Initialize broken link service
linkService, err := brokenlinks.NewService(wpClient, linkStore, cfg.LinkChecker, log)
if err != nil {
log.Error("Failed to initialize broken link service", "error", err)
os.Exit(1)
}
imageAuditService, err := imageaudit.NewService(wpClient, imageAuditStore, cfg.ImageAudit, log)
if err != nil {
log.Error("Failed to initialize image audit service", "error", err)
os.Exit(1)
}
translationService := translation.NewService(wpClient, translationClient, log)
// Initialize optimizer
optimizer := seo.NewOptimizer(wpClient, llmClient, cfg.LLM.Provider, store, cfg, log)
log.Info("Optimizer initialized")
@@ -148,7 +199,7 @@ func main() {
// Start API server if enabled
var apiServer *api.Server
if cfg.API.Enabled {
apiServer = api.NewServer(optimizer, wpClient, store, auditStore, ideaService, newsService, version, cfg.API.Port, log)
apiServer = api.NewServer(optimizer, wpClient, store, auditStore, ideaService, newsService, linkService, imageAuditService, translationService, version, cfg.API.Port, log)
// Start server in goroutine
go func() {
@@ -188,7 +239,7 @@ func main() {
// Start idea scheduler if enabled
var ideaSched *scheduler.IdeaScheduler
if cfg.Ideas.Enabled || cfg.Outreach.Enabled {
ideaSched = scheduler.NewIdeaScheduler(ideaService, cfg.Ideas.IntervalHours, cfg.Ideas.Enabled, cfg.Outreach.Enabled, log)
ideaSched = scheduler.NewIdeaScheduler(ideaService, cfg.Ideas.IntervalHours, cfg.Ideas.Enabled, cfg.Outreach.Enabled, auditStore, log)
go func() {
if err := ideaSched.Start(ctx); err != nil && err != context.Canceled {
log.Error("Idea scheduler failed", "error", err)
@@ -200,7 +251,7 @@ func main() {
// Start newsletter scheduler if enabled
var newsSched *scheduler.NewsletterScheduler
if cfg.Newsletter.Enabled {
newsSched = scheduler.NewNewsletterScheduler(newsService, cfg.Newsletter.IntervalHours, log)
newsSched = scheduler.NewNewsletterScheduler(newsService, cfg.Newsletter.IntervalHours, auditStore, log)
go func() {
if err := newsSched.Start(ctx); err != nil && err != context.Canceled {
log.Error("Newsletter scheduler failed", "error", err)
@@ -209,6 +260,30 @@ func main() {
log.Info("Newsletter scheduler started", "interval", cfg.Newsletter.IntervalHours)
}
// Start broken link scheduler if enabled
var linkSched *scheduler.BrokenLinkScheduler
if cfg.LinkChecker.Enabled {
linkSched = scheduler.NewBrokenLinkScheduler(linkService, cfg.LinkChecker.IntervalHours, auditStore, log)
go func() {
if err := linkSched.Start(ctx); err != nil && err != context.Canceled {
log.Error("Broken link scheduler failed", "error", err)
}
}()
log.Info("Broken link scheduler started", "interval", cfg.LinkChecker.IntervalHours)
}
// Start image audit scheduler if enabled
var imageAuditSched *scheduler.ImageAuditScheduler
if cfg.ImageAudit.Enabled {
imageAuditSched = scheduler.NewImageAuditScheduler(imageAuditService, cfg.ImageAudit.IntervalHours, auditStore, log)
go func() {
if err := imageAuditSched.Start(ctx); err != nil && err != context.Canceled {
log.Error("Image audit scheduler failed", "error", err)
}
}()
log.Info("Image audit scheduler started", "interval", cfg.ImageAudit.IntervalHours)
}
// Wait for interrupt signal
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
@@ -250,5 +325,15 @@ func main() {
log.Info("Newsletter scheduler stopped")
}
if linkSched != nil {
linkSched.Stop()
log.Info("Broken link scheduler stopped")
}
if imageAuditSched != nil {
imageAuditSched.Stop()
log.Info("Image audit scheduler stopped")
}
log.Info("SEO Optimizer stopped gracefully")
}
+51
View File
@@ -192,6 +192,57 @@ newsletter:
# Directory to store newsletter drafts
base_path: "./data/newsletters"
link_checker:
# Enable/disable broken link scanning
enabled: true
# Hours between broken link scans (default: 168 = weekly)
interval_hours: 168
# Timeout per link request
request_timeout: 10s
# Max posts to scan (0 = all)
max_posts: 0
# Max links per post to scan (0 = all)
max_links_per_post: 0
# Number of concurrent link checks
concurrency: 6
# Directory to store broken link reports
base_path: "./data/broken-links"
image_audit:
# Enable/disable external image audit
enabled: true
# Hours between image audits (default: 168 = weekly)
interval_hours: 168
# Base URL for resolving relative image URLs
base_url: ""
# Directory to store image audit reports
base_path: "./data/image-audits"
# Max posts to scan (0 = all)
max_posts: 0
# Max images per post to scan (0 = all)
max_images_per_post: 0
# Allowed external image host suffixes (subdomains OK)
allowed_host_suffixes:
- "optimole.com"
# Timeout for image downloads when fixing
download_timeout: 20s
# Max bytes to download per image when fixing
max_download_bytes: 20971520
storage:
# Directory to store optimization records
# This stores pending drafts and optimization history
+6 -2
View File
@@ -13,8 +13,7 @@ services:
# WordPress credentials
- WP_APP_PASSWORD=${WP_APP_PASSWORD}
# LLM provider (claude or deepseek)
- LLM_PROVIDER=${LLM_PROVIDER:-claude}
# LLM provider (claude or deepseek) - set LLM_PROVIDER to override config
# Claude API key
- CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
@@ -78,6 +77,8 @@ services:
volumes:
# Mount config file (optional, can use env vars instead)
- ./configs/config.yaml:/app/configs/config.yaml:ro
# Persist data across restarts/upgrades
- seo-optimizer-data:/app/data
healthcheck:
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/api/v1/health"]
interval: 30s
@@ -89,3 +90,6 @@ services:
options:
max-size: "10m"
max-file: "3"
volumes:
seo-optimizer-data:
+102
View File
@@ -359,6 +359,86 @@ Delete a newsletter draft.
POST /newsletter/{id}/delete
```
---
### 19. Broken Links: Latest Report
Fetch the latest broken link report (only posts with broken links are returned).
```bash
GET /broken-links
```
---
### 20. Broken Links: Scan Now
Trigger a broken link scan in the background.
```bash
POST /broken-links/scan
```
---
### 21. Image Audit: Latest Report
Fetch the latest image audit report (only posts with external images are returned).
```bash
GET /image-audit
```
---
### 22. Image Audit: Scan Now
Trigger an image audit in the background.
```bash
POST /image-audit/scan
```
---
### 23. Image Audit: Fix External Images
Create a draft that replaces external images with locally hosted AVIF uploads.
```bash
POST /image-audit/fix {"post_id": 123}
```
---
### 24. Image Audit: Sync Featured Image
Sync the English featured image to the Spanish translation.
```bash
POST /image-audit/sync-featured {"post_id": 123, "target_lang": "es"}
```
---
### 25. Orphan Posts: List
List posts missing a Spanish translation.
```bash
GET /orphans?source=en&target=es
```
---
### 26. Translate Post: Generate Draft
Generate a Spanish draft linked via Polylang.
```bash
POST /translate {"post_id": 123, "target_lang": "es"}
```
## Workflow
### Typical Optimization Workflow
@@ -441,6 +521,22 @@ Each file contains the complete optimization record including:
- Internal links
- Validation results
Broken link reports are stored at the configured `link_checker.base_path`:
```
./data/broken-links/
├── 1737730000_1737730000.json
└── 1737816400_1737816400.json
```
Image audit reports are stored at the configured `image_audit.base_path`:
```
./data/image-audits/
├── 1737730000_1737730000.json
└── 1737816400_1737816400.json
```
---
## Audit Log
@@ -487,6 +583,12 @@ storage:
audit:
base_path: "./data/audit"
link_checker:
base_path: "./data/broken-links"
image_audit:
base_path: "./data/image-audits"
```
---
+1
View File
@@ -19,6 +19,7 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.28.0 // indirect
)
+2
View File
@@ -40,6 +40,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+3
View File
@@ -97,6 +97,7 @@ func (c *ClaudeClient) OptimizePost(
c.logger.Info("Optimizing post with Claude",
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
"language", post.Lang,
)
@@ -158,6 +159,8 @@ func (c *ClaudeClient) OptimizePost(
// Log success with metrics
c.logger.Info("Claude optimization completed",
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
"duration", duration,
"input_tokens", claudeResp.Usage.InputTokens,
"output_tokens", claudeResp.Usage.OutputTokens,
+44
View File
@@ -86,6 +86,7 @@ func (c *DeepSeekClient) OptimizePost(
c.logger.Info("Optimizing post with DeepSeek",
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
"language", post.Lang,
)
@@ -131,6 +132,8 @@ func (c *DeepSeekClient) OptimizePost(
c.logger.Info("DeepSeek optimization completed",
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
"duration", duration,
"input_tokens", resp.Usage.PromptTokens,
"output_tokens", resp.Usage.CompletionTokens,
@@ -225,6 +228,47 @@ func (c *DeepSeekClient) GenerateArticleDraft(ctx context.Context, prompt string
return draft, nil
}
// TranslatePost requests a translated draft from DeepSeek.
func (c *DeepSeekClient) TranslatePost(ctx context.Context, prompt string) (*models.TranslationDraft, error) {
req := deepSeekRequest{
Model: c.model,
Messages: []deepSeekMessage{{Role: "user", Content: prompt}},
MaxTokens: c.maxTokens,
Temperature: c.temperature,
Stream: false,
}
startTime := time.Now()
resp, err := c.callAPI(ctx, req)
if err != nil {
return nil, fmt.Errorf("deepseek API call failed: %w", err)
}
duration := time.Since(startTime)
if len(resp.Choices) == 0 || strings.TrimSpace(resp.Choices[0].Message.Content) == "" {
return nil, fmt.Errorf("deepseek response has no content")
}
responseText := resp.Choices[0].Message.Content
draft, err := parseTranslationResponse(responseText, c.logger)
if err != nil {
c.logger.Error("Failed to parse DeepSeek translation response",
"error", err,
"response_preview", truncate(responseText, 500),
)
return nil, fmt.Errorf("failed to parse DeepSeek translation response: %w", err)
}
c.logger.Info("DeepSeek translation completed",
"language", draft.Language,
"duration", duration,
"input_tokens", resp.Usage.PromptTokens,
"output_tokens", resp.Usage.CompletionTokens,
)
return draft, nil
}
// GenerateOutreachSuggestions requests outreach suggestions from DeepSeek.
func (c *DeepSeekClient) GenerateOutreachSuggestions(ctx context.Context, prompt string) ([]*models.OutreachSuggestion, error) {
req := deepSeekRequest{
+46
View File
@@ -202,6 +202,39 @@ Respond with ONLY valid JSON (no markdown code blocks, no explanations outside J
- Return ONLY valid JSON, no markdown formatting, no text outside JSON
`
// TranslationPromptTemplate is used to translate a technical post into Spanish.
const TranslationPromptTemplate = `You are a senior technical editor and native Spanish writer specialized in Kubernetes, DevOps, and platform engineering.
Task: Create a Spanish version of the provided English post.
Rules:
- Keep the exact structure, headings, lists, code blocks, and HTML tags. Do NOT alter code blocks or inline code.
- Keep technical terms, product names, commands, flags, file paths, and patterns in English.
- Translate prose naturally for a native Spanish audience (neutral LATAM), improving flow and readability.
- Adapt SEO for Spanish: write a compelling SEO title (50-60 chars) and meta description (150-160 chars).
- If there are sentences that would sound unnatural in Spanish, rewrite them while keeping the meaning.
- Maintain the same overall length and section order.
Input:
Title: {{.Title}}
Slug: {{.Slug}}
SEO Title: {{.SEOTitle}}
Meta Description: {{.MetaDescription}}
Content (HTML):
{{.Content}}
Output JSON ONLY:
{
"language": "es",
"title": "string",
"slug": "string (Spanish-friendly slug, if appropriate; keep if unsure)",
"seo_title": "string (50-60 chars)",
"meta_description": "string (150-160 chars)",
"excerpt": "string (Spanish summary, 1-2 sentences)",
"content_html": "string (translated HTML, same structure)"
}
`
// RenderPrompt renders the prompt template with the provided data
func RenderPrompt(data PromptData) (string, error) {
tmpl, err := template.New("prompt").Funcs(template.FuncMap{
@@ -221,6 +254,19 @@ func RenderPrompt(data PromptData) (string, error) {
return buf.String(), nil
}
// RenderTranslationPrompt renders the translation prompt for DeepSeek.
func RenderTranslationPrompt(data PromptData) (string, error) {
tmpl, err := template.New("translation").Parse(TranslationPromptTemplate)
if err != nil {
return "", fmt.Errorf("failed to parse translation template: %w", err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, data); err != nil {
return "", fmt.Errorf("failed to execute translation template: %w", err)
}
return buf.String(), nil
}
// BuildPromptData constructs the prompt data from a WordPress post and internal posts
func BuildPromptData(
post *wordpress.Post,
+22
View File
@@ -132,6 +132,28 @@ func parseArticleDraftResponse(text string, log *logger.Logger) (*models.Article
return &draft, nil
}
func parseTranslationResponse(text string, log *logger.Logger) (*models.TranslationDraft, error) {
clean, err := extractJSON(text)
if err != nil {
return nil, err
}
var draft models.TranslationDraft
if err := json.Unmarshal([]byte(clean), &draft); err != nil {
return nil, fmt.Errorf("JSON unmarshal error: %w", err)
}
if draft.SEOTitle == "" || draft.MetaDescription == "" || draft.ContentHTML == "" {
return nil, fmt.Errorf("missing required translation fields")
}
if log != nil {
log.Debug("Parsed translation draft", "language", draft.Language, "title_length", len(draft.SEOTitle))
}
return &draft, nil
}
func extractJSON(text string) (string, error) {
text = strings.TrimSpace(text)
+316 -21
View File
@@ -14,30 +14,42 @@ import (
"github.com/google/uuid"
"github.com/gorilla/mux"
"seo-optimizer/internal/brokenlinks"
"seo-optimizer/internal/ideas"
"seo-optimizer/internal/imageaudit"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/newsletter"
"seo-optimizer/internal/seo"
"seo-optimizer/internal/storage"
"seo-optimizer/internal/translation"
"seo-optimizer/internal/wordpress"
)
// Server provides HTTP API for manual optimization triggers and status checks
type Server struct {
optimizer *seo.Optimizer
wpClient *wordpress.Client
storage *storage.OptimizationStorage
audit *storage.AuditStorage
ideaSvc *ideas.Service
newsSvc *newsletter.Service
version string
port int
server *http.Server
logger *logger.Logger
optimizer *seo.Optimizer
wpClient *wordpress.Client
storage *storage.OptimizationStorage
audit *storage.AuditStorage
ideaSvc *ideas.Service
newsSvc *newsletter.Service
linkSvc *brokenlinks.Service
imageSvc *imageaudit.Service
translationSvc *translation.Service
version string
port int
server *http.Server
logger *logger.Logger
// In-memory job tracking (simple implementation for MVP)
jobs map[string]*Job
jobsMu sync.RWMutex
brokenLinkMu sync.Mutex
brokenLinkRunning bool
imageAuditMu sync.Mutex
imageAuditRunning bool
}
// Job represents an optimization job (running or completed)
@@ -54,22 +66,25 @@ type Job struct {
}
// NewServer creates a new API server
func NewServer(optimizer *seo.Optimizer, wpClient *wordpress.Client, store *storage.OptimizationStorage, audit *storage.AuditStorage, ideaSvc *ideas.Service, newsSvc *newsletter.Service, version string, port int, log *logger.Logger) *Server {
func NewServer(optimizer *seo.Optimizer, wpClient *wordpress.Client, store *storage.OptimizationStorage, audit *storage.AuditStorage, ideaSvc *ideas.Service, newsSvc *newsletter.Service, linkSvc *brokenlinks.Service, imageSvc *imageaudit.Service, translationSvc *translation.Service, version string, port int, log *logger.Logger) *Server {
if log == nil {
log = logger.NewDefaultLogger()
}
return &Server{
optimizer: optimizer,
wpClient: wpClient,
storage: store,
audit: audit,
ideaSvc: ideaSvc,
newsSvc: newsSvc,
version: version,
port: port,
jobs: make(map[string]*Job),
logger: log.WithComponent("api"),
optimizer: optimizer,
wpClient: wpClient,
storage: store,
audit: audit,
ideaSvc: ideaSvc,
newsSvc: newsSvc,
linkSvc: linkSvc,
imageSvc: imageSvc,
translationSvc: translationSvc,
version: version,
port: port,
jobs: make(map[string]*Job),
logger: log.WithComponent("api"),
}
}
@@ -121,6 +136,14 @@ func (s *Server) Start() error {
router.HandleFunc("/api/v1/newsletter", s.handleListNewsletter).Methods("GET")
router.HandleFunc("/api/v1/newsletter/generate", s.handleGenerateNewsletter).Methods("POST")
router.HandleFunc("/api/v1/newsletter/{id}/delete", s.handleDeleteNewsletter).Methods("POST")
router.HandleFunc("/api/v1/broken-links", s.handleListBrokenLinks).Methods("GET")
router.HandleFunc("/api/v1/broken-links/scan", s.handleScanBrokenLinks).Methods("POST")
router.HandleFunc("/api/v1/image-audit", s.handleListImageAudit).Methods("GET")
router.HandleFunc("/api/v1/image-audit/scan", s.handleScanImageAudit).Methods("POST")
router.HandleFunc("/api/v1/image-audit/fix", s.handleFixExternalImages).Methods("POST")
router.HandleFunc("/api/v1/image-audit/sync-featured", s.handleSyncFeaturedImage).Methods("POST")
router.HandleFunc("/api/v1/orphans", s.handleListOrphans).Methods("GET")
router.HandleFunc("/api/v1/translate", s.handleTranslatePost).Methods("POST")
uiServer, err := uiFileServer()
if err == nil {
@@ -900,6 +923,268 @@ func (s *Server) handleDeleteNewsletter(w http.ResponseWriter, r *http.Request)
})
}
// handleListBrokenLinks handles GET /api/v1/broken-links
func (s *Server) handleListBrokenLinks(w http.ResponseWriter, r *http.Request) {
if s.linkSvc == nil {
http.Error(w, "broken link service not configured", http.StatusNotImplemented)
return
}
report, err := s.linkSvc.LatestReport()
if err != nil {
http.Error(w, fmt.Sprintf("failed to fetch broken link report: %v", err), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"report": report,
})
}
// handleScanBrokenLinks handles POST /api/v1/broken-links/scan
func (s *Server) handleScanBrokenLinks(w http.ResponseWriter, r *http.Request) {
if s.linkSvc == nil {
http.Error(w, "broken link service not configured", http.StatusNotImplemented)
return
}
s.brokenLinkMu.Lock()
if s.brokenLinkRunning {
s.brokenLinkMu.Unlock()
http.Error(w, "broken link scan already running", http.StatusConflict)
return
}
s.brokenLinkRunning = true
s.brokenLinkMu.Unlock()
requestedBy := clientFromRequest(r)
go func() {
defer func() {
s.brokenLinkMu.Lock()
s.brokenLinkRunning = false
s.brokenLinkMu.Unlock()
}()
report, err := s.linkSvc.RunScan(context.Background())
if err != nil {
s.logger.Error("Broken link scan failed", "error", err)
s.recordAudit(nil, &storage.AuditRecord{
Actor: requestedBy,
Action: "broken_links_scan_failed",
Summary: fmt.Sprintf("Broken link scan failed: %v", err),
})
return
}
s.recordAudit(nil, &storage.AuditRecord{
Actor: requestedBy,
Action: "broken_links_scanned",
Summary: fmt.Sprintf("Broken link scan completed (%d broken links)", report.TotalBrokenLinks),
})
}()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{
"status": "accepted",
})
}
// handleListImageAudit handles GET /api/v1/image-audit
func (s *Server) handleListImageAudit(w http.ResponseWriter, r *http.Request) {
if s.imageSvc == nil {
http.Error(w, "image audit service not configured", http.StatusNotImplemented)
return
}
report, err := s.imageSvc.LatestReport()
if err != nil {
http.Error(w, fmt.Sprintf("failed to fetch image audit report: %v", err), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"report": report,
})
}
// handleScanImageAudit handles POST /api/v1/image-audit/scan
func (s *Server) handleScanImageAudit(w http.ResponseWriter, r *http.Request) {
if s.imageSvc == nil {
http.Error(w, "image audit service not configured", http.StatusNotImplemented)
return
}
s.imageAuditMu.Lock()
if s.imageAuditRunning {
s.imageAuditMu.Unlock()
http.Error(w, "image audit already running", http.StatusConflict)
return
}
s.imageAuditRunning = true
s.imageAuditMu.Unlock()
requestedBy := clientFromRequest(r)
go func() {
defer func() {
s.imageAuditMu.Lock()
s.imageAuditRunning = false
s.imageAuditMu.Unlock()
}()
report, err := s.imageSvc.RunAudit(context.Background())
if err != nil {
s.logger.Error("Image audit failed", "error", err)
s.recordAudit(nil, &storage.AuditRecord{
Actor: requestedBy,
Action: "image_audit_failed",
Summary: fmt.Sprintf("Image audit failed: %v", err),
})
return
}
s.recordAudit(nil, &storage.AuditRecord{
Actor: requestedBy,
Action: "image_audit_completed",
Summary: fmt.Sprintf("Image audit completed (%d external images)", report.TotalExternalImages),
})
}()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{
"status": "accepted",
})
}
// handleFixExternalImages handles POST /api/v1/image-audit/fix
// Request: {"post_id": 123}
func (s *Server) handleFixExternalImages(w http.ResponseWriter, r *http.Request) {
if s.imageSvc == nil {
http.Error(w, "image audit service not configured", http.StatusNotImplemented)
return
}
var req struct {
PostID int `json:"post_id"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid request body", http.StatusBadRequest)
return
}
if req.PostID <= 0 {
http.Error(w, "post_id must be positive", http.StatusBadRequest)
return
}
draftID, replacements, err := s.imageSvc.FixExternalImages(r.Context(), req.PostID)
if err != nil {
http.Error(w, fmt.Sprintf("failed to fix external images: %v", err), http.StatusInternalServerError)
return
}
s.recordAudit(r, &storage.AuditRecord{
Action: "image_audit_fix_created",
Summary: fmt.Sprintf("Created draft %d to fix external images for post %d", draftID, req.PostID),
PostID: req.PostID,
})
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"draft_post_id": draftID,
"replacements": replacements,
})
}
// handleSyncFeaturedImage handles POST /api/v1/image-audit/sync-featured
// Request: {"post_id": 123, "target_lang": "es"}
func (s *Server) handleSyncFeaturedImage(w http.ResponseWriter, r *http.Request) {
if s.imageSvc == nil {
http.Error(w, "image audit service not configured", http.StatusNotImplemented)
return
}
var req struct {
PostID int `json:"post_id"`
TargetLang string `json:"target_lang"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid request body", http.StatusBadRequest)
return
}
if req.PostID <= 0 {
http.Error(w, "post_id must be positive", http.StatusBadRequest)
return
}
targetID, err := s.imageSvc.SyncFeaturedImage(r.Context(), req.PostID, req.TargetLang)
if err != nil {
http.Error(w, fmt.Sprintf("failed to sync featured image: %v", err), http.StatusInternalServerError)
return
}
s.recordAudit(r, &storage.AuditRecord{
Action: "featured_image_synced",
Summary: fmt.Sprintf("Synced featured image from post %d to post %d", req.PostID, targetID),
PostID: req.PostID,
})
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"status": "synced",
"target_post_id": targetID,
"source_post_id": req.PostID,
"target_language": req.TargetLang,
})
}
// handleListOrphans handles GET /api/v1/orphans?source=en&target=es
func (s *Server) handleListOrphans(w http.ResponseWriter, r *http.Request) {
if s.translationSvc == nil {
http.Error(w, "translation service not configured", http.StatusNotImplemented)
return
}
source := r.URL.Query().Get("source")
target := r.URL.Query().Get("target")
orphans, err := s.translationSvc.ListOrphanPosts(r.Context(), source, target)
if err != nil {
http.Error(w, fmt.Sprintf("failed to list orphan posts: %v", err), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"count": len(orphans),
"orphans": orphans,
})
}
// handleTranslatePost handles POST /api/v1/translate
// Request: {"post_id": 123, "target_lang": "es"}
func (s *Server) handleTranslatePost(w http.ResponseWriter, r *http.Request) {
if s.translationSvc == nil {
http.Error(w, "translation service not configured", http.StatusNotImplemented)
return
}
var req struct {
PostID int `json:"post_id"`
TargetLang string `json:"target_lang"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid request body", http.StatusBadRequest)
return
}
if req.PostID <= 0 {
http.Error(w, "post_id must be positive", http.StatusBadRequest)
return
}
draftID, err := s.translationSvc.TranslatePost(r.Context(), req.PostID, req.TargetLang)
if err != nil {
http.Error(w, fmt.Sprintf("failed to translate post: %v", err), http.StatusInternalServerError)
return
}
s.recordAudit(r, &storage.AuditRecord{
Action: "translation_draft_created",
Summary: fmt.Sprintf("Created translation draft %d for post %d", draftID, req.PostID),
PostID: req.PostID,
})
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"draft_post_id": draftID,
})
}
// handleGetDraft handles GET /api/v1/drafts/{id}
func (s *Server) handleGetDraft(w http.ResponseWriter, r *http.Request) {
idStr := mux.Vars(r)["id"]
@@ -995,6 +1280,10 @@ func (s *Server) handleGetComparison(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set("Content-Type", "application/json")
baseURL := ""
if s.wpClient != nil {
baseURL = s.wpClient.BaseURL()
}
json.NewEncoder(w).Encode(map[string]interface{}{
"draft": map[string]interface{}{
"id": draftPost.ID,
@@ -1002,6 +1291,7 @@ func (s *Server) handleGetComparison(w http.ResponseWriter, r *http.Request) {
"content": draftPost.Content.Rendered,
"seo_title": draftSEOTitle,
"seo_description": draftSEODescription,
"slug": draftPost.Slug,
},
"original": map[string]interface{}{
"id": originalPost.ID,
@@ -1009,6 +1299,11 @@ func (s *Server) handleGetComparison(w http.ResponseWriter, r *http.Request) {
"content": originalPost.Content.Rendered,
"seo_title": originalSEOTitle,
"seo_description": originalSEODescription,
"slug": originalPost.Slug,
},
"links": map[string]interface{}{
"draft_edit": fmt.Sprintf("%s/wp-admin/post.php?post=%d&action=edit", baseURL, draftPost.ID),
"original_edit": fmt.Sprintf("%s/wp-admin/post.php?post=%d&action=edit", baseURL, originalPost.ID),
},
})
}
+307 -3
View File
@@ -22,6 +22,9 @@ const state = {
outreach: [],
newsletters: [],
audit: [],
brokenLinks: null,
imageAudit: null,
orphans: [],
};
const appendOutput = (text) => {
@@ -103,6 +106,7 @@ const modalState = {
const renderModal = () => {
const body = byId("modal-body");
const links = byId("modal-links");
body.innerHTML = "";
const fullBtn = byId("modal-mode-full");
const compareBtn = byId("modal-mode-compare");
@@ -114,6 +118,10 @@ const renderModal = () => {
if (toggle) {
toggle.classList.add("hidden");
}
if (links) {
links.classList.add("hidden");
links.innerHTML = "";
}
const pre = document.createElement("pre");
pre.className = "modal-text";
pre.textContent = modalState.data?.body || "";
@@ -127,6 +135,18 @@ const renderModal = () => {
const data = modalState.data || {};
const draft = data.draft || {};
const original = data.original || {};
const linkData = data.links || {};
if (links) {
const parts = [];
if (linkData.draft_edit) {
parts.push(`<a href="${linkData.draft_edit}" target="_blank" rel="noreferrer">Draft in WP Editor</a>`);
}
if (linkData.original_edit) {
parts.push(`<a href="${linkData.original_edit}" target="_blank" rel="noreferrer">Original in WP Editor</a>`);
}
links.innerHTML = parts.join(" ");
links.classList.toggle("hidden", parts.length === 0);
}
if (modalState.mode === "full") {
const seoBlock = document.createElement("div");
@@ -281,6 +301,165 @@ const renderNewsletters = () => {
});
};
const renderBrokenLinks = () => {
const list = byId("broken-links-list");
const summary = byId("broken-links-summary");
list.innerHTML = "";
const report = state.brokenLinks?.report;
if (!report) {
summary.textContent = "No broken link report yet.";
return;
}
summary.textContent = `Last scan: ${new Date(report.created_at).toLocaleString()} • Posts checked: ${report.total_posts} • Links checked: ${report.links_checked} • Broken links: ${report.total_broken_links}`;
if (!report.posts || report.posts.length === 0) {
summary.textContent += " • No broken links found.";
return;
}
(report.posts || []).forEach((post) => {
const el = document.createElement("div");
el.className = "item";
const brokenItems = (post.broken_links || [])
.map((link) => {
const badgeClass = link.internal ? "badge internal" : "badge external";
const errorClass = "badge error";
const badge = `<span class=\"${badgeClass}\">${link.internal ? "Internal" : "External"}</span>`;
const status = `<span class=\"${errorClass}\">${link.status || link.status_code || "error"}</span>`;
return `
<div class=\"broken-link\">
<div>${badge} ${status}</div>
<a href=\"${link.url}\" target=\"_blank\" rel=\"noreferrer\">${link.url}</a>
</div>
`;
})
.join("");
const postLink = post.post_url
? `<a href=\"${post.post_url}\" target=\"_blank\" rel=\"noreferrer\">Open post</a>`
: "";
el.innerHTML = `
<h3>${post.post_title || "Untitled"}</h3>
<div class="meta">Post ID: ${post.post_id} Broken: ${post.total_broken} (Internal ${post.internal_broken}, External ${post.external_broken})</div>
${postLink ? `<div class=\"actions\">${postLink}</div>` : ""}
<div class=\"broken-links\">${brokenItems}</div>
`;
list.appendChild(el);
});
};
const renderImageAudit = () => {
const summary = byId("image-audit-summary");
const externalList = byId("image-audit-external-list");
const featuredList = byId("image-audit-featured-list");
externalList.innerHTML = "";
featuredList.innerHTML = "";
const report = state.imageAudit?.report;
if (!report) {
summary.textContent = "No image audit report yet.";
return;
}
summary.textContent = `Last scan: ${new Date(report.created_at).toLocaleString()} • Posts checked: ${report.total_posts} • Images checked: ${report.images_checked} • External images: ${report.total_external_images}`;
if (!report.posts || report.posts.length === 0) {
const empty = document.createElement("div");
empty.className = "meta muted";
empty.textContent = "No external image issues found.";
externalList.appendChild(empty);
} else {
report.posts.forEach((post) => {
const el = document.createElement("div");
el.className = "item";
const metaScore = post.meta
? `SEO Title Score: ${post.meta.title_score} • Description Score: ${post.meta.description_score}`
: "SEO meta not available";
const issues = post.meta
? [...(post.meta.title_issues || []), ...(post.meta.description_issues || [])].join(", ")
: "";
const images = (post.external_images || [])
.map(
(img) => `
<div class="broken-link">
<div><span class="badge external">External</span> <span class="badge error">${img.host}</span></div>
<a href="${img.url}" target="_blank" rel="noreferrer">${img.url}</a>
</div>
`
)
.join("");
el.innerHTML = `
<h3>${post.post_title || "Untitled"}</h3>
<div class="meta">Post ID: ${post.post_id} External images: ${post.external_count}</div>
<div class="meta">${metaScore}${issues ? ` • Issues: ${issues}` : ""}</div>
<div class="actions">
${post.post_url ? `<a href="${post.post_url}" target="_blank" rel="noreferrer">Open post</a>` : ""}
<button data-image-fix="${post.post_id}">Fix (create draft)</button>
</div>
<div class="broken-links">${images}</div>
`;
externalList.appendChild(el);
});
}
if (!report.featured_image_mismatches || report.featured_image_mismatches.length === 0) {
const empty = document.createElement("div");
empty.className = "meta muted";
empty.textContent = "No featured image mismatches found.";
featuredList.appendChild(empty);
} else {
report.featured_image_mismatches.forEach((item) => {
const el = document.createElement("div");
el.className = "item";
el.innerHTML = `
<h3>${item.english_title || "English Post"}</h3>
<div class="meta">EN Post ${item.english_post_id} ES Post ${item.spanish_post_id}</div>
<div class="meta">EN Featured ID: ${item.english_featured_id} ES Featured ID: ${item.spanish_featured_id}</div>
<div class="actions">
${item.english_url ? `<a href="${item.english_url}" target="_blank" rel="noreferrer">Open EN</a>` : ""}
${item.spanish_url ? `<a href="${item.spanish_url}" target="_blank" rel="noreferrer">Open ES</a>` : ""}
<button data-featured-sync="${item.english_post_id}">Sync EN ES</button>
</div>
`;
featuredList.appendChild(el);
});
}
};
const renderOrphans = () => {
const list = byId("orphans-list");
const summary = byId("orphans-summary");
list.innerHTML = "";
summary.textContent = `Orphan posts: ${state.orphans.length}`;
if (state.orphans.length === 0) {
const empty = document.createElement("div");
empty.className = "meta muted";
empty.textContent = "No orphan posts found.";
list.appendChild(empty);
return;
}
state.orphans.forEach((item) => {
const el = document.createElement("div");
el.className = "item";
el.innerHTML = `
<h3>${item.title || "Untitled"}</h3>
<div class="meta">Post ID: ${item.post_id} Missing: ${item.missing_lang}</div>
<div class="actions">
${item.url ? `<a href="${item.url}" target="_blank" rel="noreferrer">Open post</a>` : ""}
<button data-translate="${item.post_id}">Generate ES Draft</button>
</div>
`;
list.appendChild(el);
});
};
const renderAudit = () => {
const list = byId("audit-list");
list.innerHTML = "";
@@ -355,6 +534,24 @@ const loadNewsletters = async () => {
renderNewsletters();
};
const loadBrokenLinks = async () => {
const res = await apiFetch("/api/v1/broken-links");
state.brokenLinks = res || {};
renderBrokenLinks();
};
const loadImageAudit = async () => {
const res = await apiFetch("/api/v1/image-audit");
state.imageAudit = res || {};
renderImageAudit();
};
const loadOrphans = async () => {
const res = await apiFetch("/api/v1/orphans?source=en&target=es");
state.orphans = res.orphans || [];
renderOrphans();
};
const loadAudit = async () => {
const res = await apiFetch("/api/v1/audit?limit=100");
state.audit = res.records || [];
@@ -362,12 +559,22 @@ const loadAudit = async () => {
};
const refreshAll = async () => {
await Promise.all([loadHealth(), loadPending(), loadIdeas(), loadOutreach(), loadNewsletters(), loadAudit()]);
await Promise.all([
loadHealth(),
loadPending(),
loadIdeas(),
loadOutreach(),
loadNewsletters(),
loadBrokenLinks(),
loadImageAudit(),
loadOrphans(),
loadAudit(),
]);
};
const commandHandlers = {
help: async () =>
"Commands: health, version, pending [summary|text <id>], approve <id>, reject <id>, ideas [status|generate|draft <id>|publish <id>|delete <id>], outreach [status|generate|delete <id>], newsletter [status|generate|delete <id>], optimize <post_id> [lang], changes <post_id>, status <job_id>, audit [limit]",
"Commands: health, version, pending [summary|text <id>], approve <id>, reject <id>, ideas [status|generate|draft <id>|publish <id>|delete <id>], outreach [status|generate|delete <id>], newsletter [status|generate|delete <id>], broken-links [scan], image-audit [scan|fix <post_id>|sync <post_id> [lang]], orphans, translate <post_id> [lang], optimize <post_id> [lang], changes <post_id>, status <job_id>, audit [limit]",
health: async () => JSON.stringify(await apiFetch("/api/v1/health"), null, 2),
version: async () => JSON.stringify(await apiFetch("/api/v1/health"), null, 2),
pending: async (args) => {
@@ -429,6 +636,52 @@ const commandHandlers = {
const status = args[0] || "draft";
return JSON.stringify(await apiFetch(`/api/v1/newsletter?status=${status}`), null, 2);
},
"broken-links": async (args) => {
if (args[0] === "scan") {
await apiFetch("/api/v1/broken-links/scan", { method: "POST" });
return "Broken link scan started.";
}
return JSON.stringify(await apiFetch("/api/v1/broken-links"), null, 2);
},
"image-audit": async (args) => {
if (args[0] === "scan") {
await apiFetch("/api/v1/image-audit/scan", { method: "POST" });
return "Image audit started.";
}
if (args[0] === "fix" && args[1]) {
return JSON.stringify(
await apiFetch("/api/v1/image-audit/fix", {
method: "POST",
body: JSON.stringify({ post_id: Number(args[1]) }),
}),
null,
2
);
}
if (args[0] === "sync" && args[1]) {
return JSON.stringify(
await apiFetch("/api/v1/image-audit/sync-featured", {
method: "POST",
body: JSON.stringify({ post_id: Number(args[1]), target_lang: args[2] || "es" }),
}),
null,
2
);
}
return JSON.stringify(await apiFetch("/api/v1/image-audit"), null, 2);
},
orphans: async () => JSON.stringify(await apiFetch("/api/v1/orphans?source=en&target=es"), null, 2),
translate: async (args) => {
if (!args[0]) return "translate <post_id> [lang]";
return JSON.stringify(
await apiFetch("/api/v1/translate", {
method: "POST",
body: JSON.stringify({ post_id: Number(args[0]), target_lang: args[1] || "es" }),
}),
null,
2
);
},
optimize: async (args) => {
const payload = { post_id: Number(args[0]), language: args[1] || "en" };
return JSON.stringify(
@@ -459,7 +712,7 @@ const runCommand = async () => {
try {
const res = await handler(args);
setOutput(res);
await Promise.all([loadPending(), loadIdeas(), loadOutreach(), loadNewsletters(), loadAudit()]);
await Promise.all([loadPending(), loadIdeas(), loadOutreach(), loadNewsletters(), loadBrokenLinks(), loadImageAudit(), loadOrphans(), loadAudit()]);
} catch (err) {
appendOutput(`Error: ${err.message}`);
}
@@ -489,6 +742,19 @@ const wireActions = () => {
byId("refresh-ideas").addEventListener("click", loadIdeas);
byId("refresh-outreach").addEventListener("click", loadOutreach);
byId("refresh-newsletter").addEventListener("click", loadNewsletters);
byId("refresh-broken-links").addEventListener("click", loadBrokenLinks);
byId("scan-broken-links").addEventListener("click", async () => {
setOutput("Starting broken link scan...");
await apiFetch("/api/v1/broken-links/scan", { method: "POST" });
await loadBrokenLinks();
});
byId("refresh-image-audit").addEventListener("click", loadImageAudit);
byId("scan-image-audit").addEventListener("click", async () => {
setOutput("Starting image audit...");
await apiFetch("/api/v1/image-audit/scan", { method: "POST" });
await loadImageAudit();
});
byId("refresh-orphans").addEventListener("click", loadOrphans);
byId("refresh-audit").addEventListener("click", loadAudit);
byId("command-run").addEventListener("click", runCommand);
byId("command-input").addEventListener("keydown", (e) => {
@@ -584,6 +850,44 @@ const wireActions = () => {
}
});
byId("image-audit-external-list").addEventListener("click", async (e) => {
const fix = e.target.getAttribute("data-image-fix");
if (!fix) return;
setOutput(`Creating draft to fix external images for post ${fix}...`);
await apiFetch("/api/v1/image-audit/fix", {
method: "POST",
body: JSON.stringify({ post_id: Number(fix) }),
});
await loadPending();
await loadImageAudit();
await loadAudit();
});
byId("image-audit-featured-list").addEventListener("click", async (e) => {
const sync = e.target.getAttribute("data-featured-sync");
if (!sync) return;
setOutput(`Syncing featured image from EN post ${sync} to ES...`);
await apiFetch("/api/v1/image-audit/sync-featured", {
method: "POST",
body: JSON.stringify({ post_id: Number(sync), target_lang: "es" }),
});
await loadImageAudit();
await loadAudit();
});
byId("orphans-list").addEventListener("click", async (e) => {
const translate = e.target.getAttribute("data-translate");
if (!translate) return;
setOutput(`Generating Spanish draft for post ${translate}...`);
await apiFetch("/api/v1/translate", {
method: "POST",
body: JSON.stringify({ post_id: Number(translate), target_lang: "es" }),
});
await loadPending();
await loadOrphans();
await loadAudit();
});
byId("modal-close").addEventListener("click", hideModal);
byId("modal-mode-full").addEventListener("click", () => {
modalState.mode = "full";
+144 -51
View File
@@ -9,9 +9,12 @@
<body>
<div class="app">
<header class="header">
<div>
<h1>WPSideKick</h1>
<div class="sub">Server: <span id="server-version">-</span></div>
<div class="brand">
<img class="logo" src="logo.svg" alt="WPSideKick logo" />
<div>
<h1>WPSideKick</h1>
<div class="sub">Server: <span id="server-version">-</span></div>
</div>
</div>
<div class="actions">
<button id="refresh-all">Refresh</button>
@@ -21,64 +24,154 @@
</div>
</header>
<div id="outreach-alert" class="alert hidden"></div>
<section class="command">
<label for="command-input">Command Bar</label>
<div class="command-row">
<input id="command-input" placeholder="e.g., pending text 123 or outreach generate" />
<button id="command-run">Run</button>
</div>
<pre id="command-output" class="output"></pre>
</section>
<section class="grid">
<div class="panel">
<div class="panel-header">
<h2>Pending Drafts</h2>
<button id="refresh-pending">Refresh</button>
<div class="layout">
<aside class="sidebar">
<div class="side-card">
<h2>Quick Nav</h2>
<a href="#section-ops">Ops Queue</a>
<a href="#section-growth">Growth</a>
<a href="#section-health">Health</a>
<a href="#section-content">Content</a>
</div>
<div id="pending-list" class="list"></div>
</div>
<div class="panel">
<div class="panel-header">
<h2>Ideas</h2>
<button id="refresh-ideas">Refresh</button>
<div class="side-card">
<h2>Alerts</h2>
<div id="outreach-alert" class="alert hidden"></div>
<p class="meta muted">Live alerts appear here.</p>
</div>
<div id="ideas-list" class="list"></div>
</div>
</aside>
<div class="panel">
<div class="panel-header">
<h2>Outreach</h2>
<button id="refresh-outreach">Refresh</button>
</div>
<div id="outreach-list" class="list"></div>
</div>
<main class="content">
<section class="command">
<label for="command-input">Command Bar</label>
<div class="command-row">
<input id="command-input" placeholder="e.g., pending text 123 or outreach generate" />
<button id="command-run">Run</button>
</div>
<pre id="command-output" class="output"></pre>
</section>
<div class="panel">
<div class="panel-header">
<h2>Newsletter</h2>
<button id="refresh-newsletter">Refresh</button>
</div>
<div id="newsletter-list" class="list"></div>
</div>
<section id="section-ops" class="section">
<div class="section-header">
<h2>Ops Queue</h2>
<p>Drafts + missing translations you can action fast.</p>
</div>
<div class="card-grid">
<div class="panel">
<div class="panel-header">
<h3>Pending Drafts</h3>
<button id="refresh-pending">Refresh</button>
</div>
<div id="pending-list" class="list"></div>
</div>
<div class="panel">
<div class="panel-header">
<h2>Audit Log</h2>
<button id="refresh-audit">Refresh</button>
</div>
<div id="audit-list" class="list"></div>
</div>
</section>
<div class="panel">
<div class="panel-header">
<h3>Orphan Posts (Missing ES)</h3>
<button id="refresh-orphans">Refresh</button>
</div>
<div id="orphans-summary" class="meta muted"></div>
<div id="orphans-list" class="list"></div>
</div>
</div>
</section>
<section id="section-growth" class="section">
<div class="section-header">
<h2>Growth</h2>
<p>Ideas, outreach, and newsletters.</p>
</div>
<div class="card-grid">
<div class="panel">
<div class="panel-header">
<h3>Ideas</h3>
<button id="refresh-ideas">Refresh</button>
</div>
<div id="ideas-list" class="list"></div>
</div>
<div class="panel">
<div class="panel-header">
<h3>Outreach</h3>
<button id="refresh-outreach">Refresh</button>
</div>
<div id="outreach-list" class="list"></div>
</div>
<div class="panel">
<div class="panel-header">
<h3>Newsletter</h3>
<button id="refresh-newsletter">Refresh</button>
</div>
<div id="newsletter-list" class="list"></div>
</div>
</div>
</section>
<section id="section-health" class="section">
<div class="section-header">
<h2>Health</h2>
<p>Broken links + image audits prioritized by impact.</p>
</div>
<div class="card-grid">
<div class="panel">
<div class="panel-header">
<h3>Broken Links</h3>
<div class="panel-actions">
<button id="scan-broken-links">Scan</button>
<button id="refresh-broken-links">Refresh</button>
</div>
</div>
<div id="broken-links-summary" class="meta muted"></div>
<div id="broken-links-list" class="list"></div>
</div>
<div class="panel">
<div class="panel-header">
<h3>Image Audit</h3>
<div class="panel-actions">
<button id="scan-image-audit">Scan</button>
<button id="refresh-image-audit">Refresh</button>
</div>
</div>
<div id="image-audit-summary" class="meta muted"></div>
<div class="subsection">
<h4>External Images</h4>
<div id="image-audit-external-list" class="list"></div>
</div>
<div class="subsection">
<h4>Featured Image Mismatches (EN → ES)</h4>
<div id="image-audit-featured-list" class="list"></div>
</div>
</div>
</div>
</section>
<section id="section-content" class="section">
<div class="section-header">
<h2>Activity</h2>
<p>Recent system actions.</p>
</div>
<div class="card-grid">
<div class="panel">
<div class="panel-header">
<h3>Audit Log</h3>
<button id="refresh-audit">Refresh</button>
</div>
<div id="audit-list" class="list"></div>
</div>
</div>
</section>
</main>
</div>
</div>
<div id="modal" class="modal hidden">
<div class="modal-content">
<div class="modal-header">
<h3 id="modal-title">Draft</h3>
<div class="modal-title">
<h3 id="modal-title">Draft</h3>
<div id="modal-links" class="modal-links hidden"></div>
</div>
<div class="modal-actions">
<div class="toggle" id="modal-toggle">
<button id="modal-mode-full" class="active">Full</button>
+11
View File
@@ -0,0 +1,11 @@
<svg xmlns="http://www.w3.org/2000/svg" width="96" height="96" viewBox="0 0 96 96" role="img" aria-label="WPSideKick logo">
<defs>
<linearGradient id="wpsk-grad" x1="0" y1="0" x2="1" y2="1">
<stop offset="0%" stop-color="#4ec9b0" />
<stop offset="100%" stop-color="#6aa9ff" />
</linearGradient>
</defs>
<rect x="10" y="10" width="76" height="76" rx="18" fill="#12151c" stroke="url(#wpsk-grad)" stroke-width="3" />
<path d="M28 60l6-24h8l6 24 6-24h8l-10 36h-8l-6-22-6 22h-8l-10-36h8l6 24z" fill="url(#wpsk-grad)" />
<circle cx="72" cy="26" r="8" fill="#f0b429" opacity="0.9" />
</svg>

After

Width:  |  Height:  |  Size: 614 B

+340 -66
View File
@@ -1,11 +1,16 @@
@import url("https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,500;9..144,700&family=Space+Grotesk:wght@400;500;600&display=swap");
:root {
--bg: #0f1115;
--panel: #171a21;
--text: #f4f4f4;
--muted: #9aa3b2;
--accent: #4ec9b0;
--warn: #f0b429;
--danger: #ff6b6b;
--bg: #f7f3ee;
--panel: #fffaf3;
--text: #1b1b1b;
--muted: #6d6a63;
--accent: #1f6f5b;
--accent-strong: #0e4a3c;
--warn: #d08a1f;
--danger: #b64834;
--stroke: #e3dbd2;
--shadow: 0 20px 40px rgba(23, 17, 10, 0.08);
}
* {
@@ -14,27 +19,67 @@
body {
margin: 0;
font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
background: radial-gradient(circle at top, #1b1f2a, #0f1115 60%);
font-family: "Space Grotesk", sans-serif;
color: var(--text);
background: radial-gradient(circle at top, #fffaf3 0%, #f6efe6 40%, #efe3d6 100%);
}
body::before,
body::after {
content: "";
position: fixed;
width: 420px;
height: 420px;
border-radius: 50%;
background: radial-gradient(circle, rgba(31, 111, 91, 0.18), transparent 70%);
z-index: -1;
}
body::before {
top: -120px;
right: -120px;
}
body::after {
bottom: -140px;
left: -120px;
background: radial-gradient(circle, rgba(208, 138, 31, 0.2), transparent 70%);
}
.app {
max-width: 1200px;
margin: 32px auto;
padding: 0 16px 48px;
max-width: 1400px;
margin: 32px auto 64px;
padding: 0 24px;
}
.header {
display: flex;
justify-content: space-between;
gap: 16px;
align-items: center;
gap: 24px;
padding: 20px 24px;
border-radius: 20px;
background: var(--panel);
border: 1px solid var(--stroke);
box-shadow: var(--shadow);
}
.brand {
display: flex;
align-items: center;
gap: 14px;
}
.logo {
width: 48px;
height: 48px;
}
.header h1 {
margin: 0;
font-size: 28px;
font-family: "Fraunces", serif;
font-size: 30px;
letter-spacing: -0.5px;
}
.sub {
@@ -42,17 +87,62 @@ body {
font-size: 13px;
}
.actions button {
margin-left: 8px;
.actions {
display: flex;
flex-wrap: wrap;
gap: 10px;
}
.layout {
display: grid;
grid-template-columns: minmax(220px, 260px) minmax(0, 1fr);
gap: 24px;
margin-top: 24px;
}
.sidebar {
display: grid;
gap: 18px;
align-content: start;
}
.side-card {
padding: 18px;
border-radius: 16px;
background: var(--panel);
border: 1px solid var(--stroke);
box-shadow: var(--shadow);
display: grid;
gap: 10px;
}
.side-card h2 {
margin: 0;
font-size: 16px;
font-weight: 600;
}
.side-card a {
color: var(--accent);
text-decoration: none;
font-weight: 500;
}
.side-card a:hover {
text-decoration: underline;
}
.content {
display: grid;
gap: 28px;
}
.alert {
margin: 16px 0;
padding: 10px 12px;
border: 1px solid var(--warn);
background: rgba(240, 180, 41, 0.15);
background: rgba(208, 138, 31, 0.12);
color: var(--warn);
border-radius: 6px;
border-radius: 8px;
}
.hidden {
@@ -60,75 +150,111 @@ body {
}
button {
background: var(--panel);
color: var(--text);
border: 1px solid #2a2f3a;
padding: 8px 12px;
border-radius: 6px;
background: var(--accent);
color: #fff;
border: 0;
padding: 8px 14px;
border-radius: 999px;
font-weight: 600;
cursor: pointer;
transition: transform 0.2s ease, box-shadow 0.2s ease;
}
button:hover {
border-color: var(--accent);
transform: translateY(-1px);
box-shadow: 0 12px 20px rgba(31, 111, 91, 0.15);
}
button:active {
transform: translateY(0);
}
.command {
margin-top: 24px;
padding: 16px;
padding: 18px;
border-radius: 18px;
background: var(--panel);
border-radius: 8px;
border: 1px solid #222833;
border: 1px solid var(--stroke);
box-shadow: var(--shadow);
}
.command label {
display: block;
margin-bottom: 8px;
margin-bottom: 10px;
font-weight: 600;
color: var(--muted);
}
.command-row {
display: flex;
gap: 8px;
gap: 10px;
}
.command input {
flex: 1;
padding: 10px;
border-radius: 6px;
background: #0e1016;
border: 1px solid #2a2f3a;
color: var(--text);
padding: 12px;
border-radius: 999px;
border: 1px solid var(--stroke);
background: #fff;
font-family: inherit;
}
.output {
margin-top: 12px;
background: #0e1016;
border: 1px solid #2a2f3a;
background: #fff;
border: 1px dashed var(--stroke);
padding: 12px;
border-radius: 6px;
max-height: 220px;
border-radius: 12px;
max-height: 200px;
overflow: auto;
white-space: pre-wrap;
}
.grid {
.section {
display: grid;
gap: 16px;
}
.section-header h2 {
margin: 0;
font-family: "Fraunces", serif;
font-size: 24px;
}
.section-header p {
margin: 4px 0 0;
color: var(--muted);
}
.card-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
gap: 16px;
margin-top: 20px;
}
.panel {
background: var(--panel);
border-radius: 8px;
border: 1px solid #222833;
padding: 12px;
border-radius: 18px;
border: 1px solid var(--stroke);
padding: 14px;
box-shadow: var(--shadow);
}
.panel-header {
display: flex;
justify-content: space-between;
align-items: center;
gap: 10px;
}
.panel-header h3 {
margin: 0;
font-size: 16px;
}
.panel-actions {
display: flex;
gap: 8px;
flex-wrap: wrap;
}
.list {
@@ -138,10 +264,10 @@ button:hover {
}
.item {
background: #12151c;
border: 1px solid #222833;
padding: 10px;
border-radius: 6px;
background: #fff;
border: 1px solid var(--stroke);
padding: 12px;
border-radius: 12px;
}
.item h3 {
@@ -157,22 +283,100 @@ button:hover {
.item .actions {
display: flex;
gap: 6px;
gap: 8px;
flex-wrap: wrap;
}
.item .actions a {
background: #fff;
border: 1px solid var(--stroke);
padding: 6px 12px;
border-radius: 999px;
text-decoration: none;
color: var(--accent-strong);
font-weight: 600;
}
.item .actions a:hover {
border-color: var(--accent);
}
.muted {
color: var(--muted);
}
.danger {
border-color: var(--danger);
background: transparent;
border: 1px solid var(--danger);
color: var(--danger);
}
button.danger:hover {
box-shadow: none;
transform: translateY(-1px);
}
.subsection h4 {
margin: 12px 0 6px;
font-size: 14px;
color: var(--muted);
}
.broken-links {
display: grid;
gap: 8px;
font-size: 12px;
}
.broken-link {
display: grid;
gap: 4px;
}
.broken-link a {
color: var(--accent);
text-decoration: none;
word-break: break-word;
}
.broken-link a:hover {
text-decoration: underline;
}
.badge {
display: inline-flex;
align-items: center;
gap: 4px;
font-size: 11px;
padding: 2px 8px;
border-radius: 999px;
border: 1px solid var(--stroke);
color: var(--muted);
}
.badge.internal {
color: var(--accent);
border-color: rgba(31, 111, 91, 0.4);
}
.badge.external {
color: var(--warn);
border-color: rgba(208, 138, 31, 0.4);
}
.badge.error {
color: var(--danger);
border-color: rgba(182, 72, 52, 0.4);
}
.modal {
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.6);
background: rgba(0, 0, 0, 0.5);
display: flex;
align-items: center;
justify-content: center;
z-index: 10;
}
.modal.hidden {
@@ -180,17 +384,43 @@ button:hover {
}
.modal-content {
width: min(900px, 90vw);
width: min(980px, 92vw);
background: var(--panel);
border-radius: 8px;
border: 1px solid #222833;
padding: 16px;
border-radius: 18px;
border: 1px solid var(--stroke);
padding: 18px;
box-shadow: var(--shadow);
}
.modal-header {
display: flex;
justify-content: space-between;
align-items: center;
gap: 12px;
}
.modal-title {
display: grid;
gap: 6px;
}
.modal-links {
display: flex;
gap: 10px;
flex-wrap: wrap;
font-size: 12px;
}
.modal-links a {
color: var(--accent);
text-decoration: none;
border: 1px solid var(--stroke);
padding: 4px 8px;
border-radius: 999px;
}
.modal-links a:hover {
border-color: var(--accent);
}
.modal-actions {
@@ -201,19 +431,20 @@ button:hover {
.toggle {
display: inline-flex;
border: 1px solid #2a2f3a;
border-radius: 6px;
border: 1px solid var(--stroke);
border-radius: 999px;
overflow: hidden;
}
.toggle button {
border: 0;
background: transparent;
padding: 6px 10px;
color: var(--text);
padding: 6px 12px;
}
.toggle button.active {
background: #222833;
background: #f0e6da;
color: var(--text);
}
@@ -224,24 +455,24 @@ button:hover {
}
.modal-text {
background: #0e1016;
border: 1px solid #2a2f3a;
background: #fff;
border: 1px solid var(--stroke);
padding: 12px;
border-radius: 6px;
border-radius: 12px;
max-height: 60vh;
overflow: auto;
white-space: pre-wrap;
}
.diff-add {
background: rgba(78, 201, 176, 0.2);
color: var(--accent);
background: rgba(31, 111, 91, 0.18);
color: var(--accent-strong);
border-radius: 3px;
padding: 0 2px;
}
.diff-del {
background: rgba(255, 107, 107, 0.2);
background: rgba(182, 72, 52, 0.18);
color: var(--danger);
border-radius: 3px;
padding: 0 2px;
@@ -265,8 +496,51 @@ button:hover {
margin-bottom: 8px;
}
@media (max-width: 1100px) {
.layout {
grid-template-columns: 1fr;
}
.sidebar {
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
}
}
@media (max-width: 900px) {
.header {
flex-direction: column;
align-items: flex-start;
}
.actions {
width: 100%;
}
.command-row {
flex-direction: column;
}
.compare-grid {
grid-template-columns: 1fr;
}
}
@media (max-width: 600px) {
.app {
padding: 0 16px;
}
.panel {
padding: 12px;
}
button {
width: 100%;
}
.panel-actions,
.item .actions {
flex-direction: column;
align-items: stretch;
}
}
+404
View File
@@ -0,0 +1,404 @@
package brokenlinks
import (
"context"
"errors"
"net"
"net/http"
"net/url"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/net/html"
"seo-optimizer/internal/config"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/storage"
"seo-optimizer/internal/wordpress"
"seo-optimizer/pkg/models"
)
type Service struct {
wp *wordpress.Client
store *storage.BrokenLinkStorage
logger *logger.Logger
baseURL *url.URL
httpClient *http.Client
maxPosts int
maxLinksPer int
concurrency int
}
type linkCheckResult struct {
statusCode int
status string
}
// NewService creates a broken link scanning service.
func NewService(wp *wordpress.Client, store *storage.BrokenLinkStorage, cfg config.LinkCheckerConfig, log *logger.Logger) (*Service, error) {
if log == nil {
log = logger.NewDefaultLogger()
}
baseURL, err := url.Parse(strings.TrimSpace(cfg.BaseURL))
if err != nil {
return nil, err
}
if cfg.Concurrency <= 0 {
cfg.Concurrency = 6
}
if cfg.RequestTimeout <= 0 {
cfg.RequestTimeout = 10 * time.Second
}
return &Service{
wp: wp,
store: store,
logger: log.WithComponent("broken_links"),
baseURL: baseURL,
httpClient: &http.Client{Timeout: cfg.RequestTimeout},
maxPosts: cfg.MaxPosts,
maxLinksPer: cfg.MaxLinksPerPost,
concurrency: cfg.Concurrency,
}, nil
}
// RunScan scans all posts and stores a report.
func (s *Service) RunScan(ctx context.Context) (*models.BrokenLinkReport, error) {
report, err := s.scan(ctx)
if err != nil {
return nil, err
}
if s.store != nil {
if err := s.store.Save(report); err != nil {
return report, err
}
}
return report, nil
}
// LatestReport returns the latest stored report.
func (s *Service) LatestReport() (*models.BrokenLinkReport, error) {
if s.store == nil {
return nil, nil
}
return s.store.Latest()
}
func (s *Service) scan(ctx context.Context) (*models.BrokenLinkReport, error) {
start := time.Now()
perPage := 100
page := 1
var checkedLinks atomic.Int64
processedPosts := 0
var postsWithIssues []models.PostBrokenLinks
cache := make(map[string]linkCheckResult)
cacheMu := sync.Mutex{}
for {
if s.maxPosts > 0 && processedPosts >= s.maxPosts {
break
}
params := wordpress.ListParams{
Status: "publish",
OrderBy: "modified",
Order: "desc",
PerPage: perPage,
Page: page,
}
posts, err := s.wp.ListPosts(ctx, params)
if err != nil {
return nil, err
}
if len(posts) == 0 {
break
}
for _, post := range posts {
if s.maxPosts > 0 && processedPosts >= s.maxPosts {
break
}
processedPosts += 1
content := strings.TrimSpace(post.Content.Rendered)
if content == "" {
continue
}
links := extractLinks(content)
links = filterLinks(links)
links = uniqueLinks(links)
if s.maxLinksPer > 0 && len(links) > s.maxLinksPer {
links = links[:s.maxLinksPer]
}
if len(links) == 0 {
continue
}
postIssues := models.PostBrokenLinks{
PostID: post.ID,
PostTitle: post.Title.Rendered,
PostURL: post.Link,
}
jobs := make(chan string)
results := make(chan models.BrokenLinkIssue)
var wg sync.WaitGroup
worker := func() {
defer wg.Done()
for link := range jobs {
resolved, internal, ok := s.resolveLink(link)
if !ok {
continue
}
cacheMu.Lock()
result, cached := cache[resolved]
cacheMu.Unlock()
if !cached {
result = s.checkLink(ctx, resolved)
cacheMu.Lock()
cache[resolved] = result
cacheMu.Unlock()
checkedLinks.Add(1)
}
if result.statusCode == http.StatusNotFound || result.statusCode >= 500 || result.status == "timeout" || result.status == "error" {
results <- models.BrokenLinkIssue{
URL: resolved,
Status: result.status,
StatusCode: result.statusCode,
Internal: internal,
}
}
}
}
for i := 0; i < s.concurrency; i++ {
wg.Add(1)
go worker()
}
go func() {
wg.Wait()
close(results)
}()
go func() {
for _, link := range links {
jobs <- link
}
close(jobs)
}()
for issue := range results {
postIssues.BrokenLinks = append(postIssues.BrokenLinks, issue)
postIssues.TotalBroken += 1
if issue.Internal {
postIssues.InternalBroken += 1
} else {
postIssues.ExternalBroken += 1
}
}
if postIssues.TotalBroken > 0 {
postsWithIssues = append(postsWithIssues, postIssues)
}
}
if len(posts) < perPage {
break
}
page += 1
}
sort.Slice(postsWithIssues, func(i, j int) bool {
if postsWithIssues[i].TotalBroken == postsWithIssues[j].TotalBroken {
return postsWithIssues[i].PostID < postsWithIssues[j].PostID
}
return postsWithIssues[i].TotalBroken > postsWithIssues[j].TotalBroken
})
brokenTotal := 0
for _, post := range postsWithIssues {
brokenTotal += post.TotalBroken
}
report := &models.BrokenLinkReport{
ID: "",
CreatedAt: time.Now(),
DurationMs: time.Since(start).Milliseconds(),
TotalPosts: processedPosts,
LinksChecked: int(checkedLinks.Load()),
TotalBrokenLinks: brokenTotal,
Posts: postsWithIssues,
}
return report, nil
}
func (s *Service) resolveLink(raw string) (string, bool, bool) {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return "", false, false
}
parsed, err := url.Parse(trimmed)
if err != nil {
return "", false, false
}
if parsed.Scheme != "" && parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", false, false
}
if strings.HasPrefix(trimmed, "#") {
return "", false, false
}
if parsed.Host == "" {
resolved := s.baseURL.ResolveReference(parsed)
return resolved.String(), true, true
}
if parsed.Scheme == "" && s.baseURL != nil && s.baseURL.Scheme != "" {
parsed.Scheme = s.baseURL.Scheme
}
internal := hostsMatch(s.baseURL, parsed)
return parsed.String(), internal, true
}
func (s *Service) checkLink(ctx context.Context, link string) linkCheckResult {
result := linkCheckResult{statusCode: 0, status: "error"}
if link == "" {
return result
}
headReq, err := http.NewRequestWithContext(ctx, http.MethodHead, link, nil)
if err == nil {
resp, err := s.httpClient.Do(headReq)
if err == nil {
resp.Body.Close()
result.statusCode = resp.StatusCode
result.status = statusLabel(resp.StatusCode)
if resp.StatusCode != http.StatusMethodNotAllowed && resp.StatusCode != http.StatusForbidden {
return result
}
}
}
getReq, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil)
if err != nil {
return result
}
resp, err := s.httpClient.Do(getReq)
if err != nil {
if isTimeout(err) {
return linkCheckResult{statusCode: 0, status: "timeout"}
}
return result
}
defer resp.Body.Close()
return linkCheckResult{statusCode: resp.StatusCode, status: statusLabel(resp.StatusCode)}
}
func statusLabel(code int) string {
if code == http.StatusNotFound {
return "404"
}
if code >= 500 {
return "500"
}
if code == 0 {
return "error"
}
return "ok"
}
func isTimeout(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) {
return netErr.Timeout()
}
return false
}
func extractLinks(htmlContent string) []string {
root, err := html.Parse(strings.NewReader(htmlContent))
if err != nil {
return nil
}
var links []string
var walker func(*html.Node)
walker = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "a" {
for _, attr := range n.Attr {
if strings.EqualFold(attr.Key, "href") {
links = append(links, attr.Val)
break
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walker(c)
}
}
walker(root)
return links
}
func filterLinks(links []string) []string {
filtered := make([]string, 0, len(links))
for _, link := range links {
trimmed := strings.TrimSpace(link)
if trimmed == "" {
continue
}
lower := strings.ToLower(trimmed)
switch {
case strings.HasPrefix(lower, "#"):
continue
case strings.HasPrefix(lower, "mailto:"):
continue
case strings.HasPrefix(lower, "tel:"):
continue
case strings.HasPrefix(lower, "javascript:"):
continue
case strings.HasPrefix(lower, "data:"):
continue
}
filtered = append(filtered, trimmed)
}
return filtered
}
func uniqueLinks(links []string) []string {
seen := make(map[string]struct{}, len(links))
unique := make([]string, 0, len(links))
for _, link := range links {
if _, ok := seen[link]; ok {
continue
}
seen[link] = struct{}{}
unique = append(unique, link)
}
return unique
}
func hostsMatch(base *url.URL, other *url.URL) bool {
if base == nil || other == nil {
return false
}
baseHost := strings.ToLower(strings.TrimPrefix(base.Hostname(), "www."))
otherHost := strings.ToLower(strings.TrimPrefix(other.Hostname(), "www."))
return baseHost == otherHost
}
+120
View File
@@ -21,6 +21,8 @@ type Config struct {
Ideas IdeasConfig `mapstructure:"ideas"`
Outreach OutreachConfig `mapstructure:"outreach"`
Newsletter NewsletterConfig `mapstructure:"newsletter"`
ImageAudit ImageAuditConfig `mapstructure:"image_audit"`
LinkChecker LinkCheckerConfig `mapstructure:"link_checker"`
Storage StorageConfig `mapstructure:"storage"`
Audit AuditConfig `mapstructure:"audit"`
API APIConfig `mapstructure:"api"`
@@ -126,6 +128,31 @@ type NewsletterConfig struct {
BasePath string `mapstructure:"base_path"`
}
// ImageAuditConfig holds external image audit configuration.
type ImageAuditConfig struct {
Enabled bool `mapstructure:"enabled"`
IntervalHours float64 `mapstructure:"interval_hours"`
BaseURL string `mapstructure:"base_url"`
BasePath string `mapstructure:"base_path"`
MaxPosts int `mapstructure:"max_posts"`
MaxImagesPerPost int `mapstructure:"max_images_per_post"`
AllowedHostSuffixes []string `mapstructure:"allowed_host_suffixes"`
DownloadTimeout time.Duration `mapstructure:"download_timeout"`
MaxDownloadBytes int64 `mapstructure:"max_download_bytes"`
}
// LinkCheckerConfig holds broken link scanning configuration.
type LinkCheckerConfig struct {
Enabled bool `mapstructure:"enabled"`
IntervalHours float64 `mapstructure:"interval_hours"`
RequestTimeout time.Duration `mapstructure:"request_timeout"`
MaxPosts int `mapstructure:"max_posts"`
MaxLinksPerPost int `mapstructure:"max_links_per_post"`
Concurrency int `mapstructure:"concurrency"`
BasePath string `mapstructure:"base_path"`
BaseURL string `mapstructure:"base_url"`
}
// StorageConfig holds storage configuration
type StorageConfig struct {
BasePath string `mapstructure:"base_path"`
@@ -210,6 +237,23 @@ func LoadConfig(configPath string) (*Config, error) {
viper.BindEnv("newsletter.min_score", "NEWSLETTER_MIN_SCORE")
viper.BindEnv("newsletter.min_comments", "NEWSLETTER_MIN_COMMENTS")
viper.BindEnv("newsletter.base_path", "NEWSLETTER_STORAGE_PATH")
viper.BindEnv("image_audit.enabled", "IMAGE_AUDIT_ENABLED")
viper.BindEnv("image_audit.interval_hours", "IMAGE_AUDIT_INTERVAL_HOURS")
viper.BindEnv("image_audit.base_url", "IMAGE_AUDIT_BASE_URL")
viper.BindEnv("image_audit.base_path", "IMAGE_AUDIT_BASE_PATH")
viper.BindEnv("image_audit.max_posts", "IMAGE_AUDIT_MAX_POSTS")
viper.BindEnv("image_audit.max_images_per_post", "IMAGE_AUDIT_MAX_IMAGES_PER_POST")
viper.BindEnv("image_audit.allowed_host_suffixes", "IMAGE_AUDIT_ALLOWED_HOST_SUFFIXES")
viper.BindEnv("image_audit.download_timeout", "IMAGE_AUDIT_DOWNLOAD_TIMEOUT")
viper.BindEnv("image_audit.max_download_bytes", "IMAGE_AUDIT_MAX_DOWNLOAD_BYTES")
viper.BindEnv("link_checker.enabled", "LINK_CHECKER_ENABLED")
viper.BindEnv("link_checker.interval_hours", "LINK_CHECKER_INTERVAL_HOURS")
viper.BindEnv("link_checker.request_timeout", "LINK_CHECKER_TIMEOUT")
viper.BindEnv("link_checker.max_posts", "LINK_CHECKER_MAX_POSTS")
viper.BindEnv("link_checker.max_links_per_post", "LINK_CHECKER_MAX_LINKS_PER_POST")
viper.BindEnv("link_checker.concurrency", "LINK_CHECKER_CONCURRENCY")
viper.BindEnv("link_checker.base_path", "LINK_CHECKER_STORAGE_PATH")
viper.BindEnv("link_checker.base_url", "LINK_CHECKER_BASE_URL")
viper.BindEnv("audit.base_path", "AUDIT_STORAGE_PATH")
viper.BindEnv("logging.level", "LOG_LEVEL")
viper.BindEnv("api.port", "API_PORT")
@@ -266,9 +310,20 @@ func LoadConfig(configPath string) (*Config, error) {
config.Ideas.BasePath = os.ExpandEnv(config.Ideas.BasePath)
config.Outreach.BasePath = os.ExpandEnv(config.Outreach.BasePath)
config.Newsletter.BasePath = os.ExpandEnv(config.Newsletter.BasePath)
config.ImageAudit.BasePath = os.ExpandEnv(config.ImageAudit.BasePath)
config.ImageAudit.BaseURL = os.ExpandEnv(config.ImageAudit.BaseURL)
config.LinkChecker.BasePath = os.ExpandEnv(config.LinkChecker.BasePath)
config.LinkChecker.BaseURL = os.ExpandEnv(config.LinkChecker.BaseURL)
config.Audit.BasePath = os.ExpandEnv(config.Audit.BasePath)
config.Notifications.WebhookURL = os.ExpandEnv(config.Notifications.WebhookURL)
if strings.TrimSpace(config.ImageAudit.BaseURL) == "" {
config.ImageAudit.BaseURL = config.WordPress.BaseURL
}
if strings.TrimSpace(config.LinkChecker.BaseURL) == "" {
config.LinkChecker.BaseURL = config.WordPress.BaseURL
}
return &config, nil
}
@@ -341,6 +396,27 @@ func setDefaults() {
viper.SetDefault("newsletter.min_comments", 2)
viper.SetDefault("newsletter.base_path", "./data/newsletters")
// Image audit defaults
viper.SetDefault("image_audit.enabled", true)
viper.SetDefault("image_audit.interval_hours", 168.0)
viper.SetDefault("image_audit.base_url", "")
viper.SetDefault("image_audit.base_path", "./data/image-audits")
viper.SetDefault("image_audit.max_posts", 0)
viper.SetDefault("image_audit.max_images_per_post", 0)
viper.SetDefault("image_audit.allowed_host_suffixes", []string{"optimole.com"})
viper.SetDefault("image_audit.download_timeout", 20*time.Second)
viper.SetDefault("image_audit.max_download_bytes", int64(20*1024*1024))
// Link checker defaults
viper.SetDefault("link_checker.enabled", true)
viper.SetDefault("link_checker.interval_hours", 168.0)
viper.SetDefault("link_checker.request_timeout", 10*time.Second)
viper.SetDefault("link_checker.max_posts", 0)
viper.SetDefault("link_checker.max_links_per_post", 0)
viper.SetDefault("link_checker.concurrency", 6)
viper.SetDefault("link_checker.base_path", "./data/broken-links")
viper.SetDefault("link_checker.base_url", "")
// Storage defaults
viper.SetDefault("storage.base_path", "./data/optimizations")
viper.SetDefault("audit.base_path", "./data/audit")
@@ -517,6 +593,50 @@ func validateConfig(cfg *Config) error {
}
}
// Validate image audit config
if cfg.ImageAudit.Enabled {
if cfg.ImageAudit.IntervalHours <= 0 {
return fmt.Errorf("image_audit.interval_hours must be positive")
}
if strings.TrimSpace(cfg.ImageAudit.BasePath) == "" {
return fmt.Errorf("image_audit.base_path is required")
}
if cfg.ImageAudit.MaxPosts < 0 {
return fmt.Errorf("image_audit.max_posts must be non-negative")
}
if cfg.ImageAudit.MaxImagesPerPost < 0 {
return fmt.Errorf("image_audit.max_images_per_post must be non-negative")
}
if cfg.ImageAudit.DownloadTimeout <= 0 {
return fmt.Errorf("image_audit.download_timeout must be positive")
}
if cfg.ImageAudit.MaxDownloadBytes <= 0 {
return fmt.Errorf("image_audit.max_download_bytes must be positive")
}
}
// Validate link checker config
if cfg.LinkChecker.Enabled {
if cfg.LinkChecker.IntervalHours <= 0 {
return fmt.Errorf("link_checker.interval_hours must be positive")
}
if cfg.LinkChecker.RequestTimeout <= 0 {
return fmt.Errorf("link_checker.request_timeout must be positive")
}
if cfg.LinkChecker.MaxPosts < 0 {
return fmt.Errorf("link_checker.max_posts must be non-negative")
}
if cfg.LinkChecker.MaxLinksPerPost < 0 {
return fmt.Errorf("link_checker.max_links_per_post must be non-negative")
}
if cfg.LinkChecker.Concurrency <= 0 {
return fmt.Errorf("link_checker.concurrency must be positive")
}
if strings.TrimSpace(cfg.LinkChecker.BasePath) == "" {
return fmt.Errorf("link_checker.base_path is required")
}
}
// Validate storage config
if cfg.Storage.BasePath == "" {
return fmt.Errorf("storage.base_path is required")
+692
View File
@@ -0,0 +1,692 @@
package imageaudit
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strings"
"time"
"golang.org/x/net/html"
"seo-optimizer/internal/config"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/storage"
"seo-optimizer/internal/wordpress"
"seo-optimizer/pkg/models"
)
type Service struct {
wp *wordpress.Client
store *storage.ImageAuditStorage
logger *logger.Logger
baseURL *url.URL
allowedSuffixes []string
maxPosts int
maxImagesPerPost int
downloadTimeout time.Duration
maxDownloadBytes int64
}
type imageRef struct {
url string
source string
}
// NewService creates a new image audit service.
func NewService(wp *wordpress.Client, store *storage.ImageAuditStorage, cfg config.ImageAuditConfig, log *logger.Logger) (*Service, error) {
if log == nil {
log = logger.NewDefaultLogger()
}
baseURL, err := url.Parse(strings.TrimSpace(cfg.BaseURL))
if err != nil {
return nil, err
}
allowed := make([]string, 0, len(cfg.AllowedHostSuffixes))
for _, suffix := range cfg.AllowedHostSuffixes {
trimmed := strings.ToLower(strings.TrimSpace(suffix))
if trimmed == "" {
continue
}
allowed = append(allowed, trimmed)
}
return &Service{
wp: wp,
store: store,
logger: log.WithComponent("image_audit"),
baseURL: baseURL,
allowedSuffixes: allowed,
maxPosts: cfg.MaxPosts,
maxImagesPerPost: cfg.MaxImagesPerPost,
downloadTimeout: cfg.DownloadTimeout,
maxDownloadBytes: cfg.MaxDownloadBytes,
}, nil
}
// RunAudit scans posts for external images, SEO meta quality, and featured image mismatches.
func (s *Service) RunAudit(ctx context.Context) (*models.ImageAuditReport, error) {
report, err := s.scan(ctx)
if err != nil {
return nil, err
}
if s.store != nil {
if err := s.store.Save(report); err != nil {
return report, err
}
}
return report, nil
}
// LatestReport returns the latest stored audit report.
func (s *Service) LatestReport() (*models.ImageAuditReport, error) {
if s.store == nil {
return nil, nil
}
return s.store.Latest()
}
// FixExternalImages downloads external images, converts to AVIF, uploads to WordPress media, and creates a draft.
func (s *Service) FixExternalImages(ctx context.Context, postID int) (int, map[string]string, error) {
post, err := s.wp.GetPostForEdit(ctx, postID, "")
if err != nil {
return 0, nil, err
}
content := strings.TrimSpace(post.Content.Raw)
if content == "" {
content = post.Content.Rendered
}
refs := extractImageRefs(content)
issueURLs := s.externalImageURLs(refs)
if len(issueURLs) == 0 {
return 0, nil, fmt.Errorf("no external images to fix")
}
replacements := make(map[string]string, len(issueURLs))
for _, raw := range issueURLs {
newURL, err := s.convertAndUpload(ctx, raw, post.Title.Rendered)
if err != nil {
return 0, replacements, err
}
replacements[raw] = newURL
content = strings.ReplaceAll(content, raw, newURL)
}
updates := wordpress.PostUpdate{
Status: "draft",
Title: post.Title.Rendered,
Content: content,
Excerpt: post.Excerpt.Rendered,
}
draftID, err := s.wp.CreateDraftRevision(ctx, postID, updates)
if err != nil {
return 0, replacements, err
}
return draftID, replacements, nil
}
// SyncFeaturedImage copies the featured image from the given post to its translation.
func (s *Service) SyncFeaturedImage(ctx context.Context, postID int, targetLang string) (int, error) {
post, err := s.wp.GetPost(ctx, postID, "")
if err != nil {
return 0, err
}
if post.FeaturedMedia == 0 {
return 0, fmt.Errorf("post %d has no featured image", postID)
}
if targetLang == "" {
targetLang = "es"
}
translatedID := 0
if post.Translations != nil {
translatedID = post.Translations[targetLang]
}
if translatedID == 0 {
return 0, fmt.Errorf("post %d has no %s translation", postID, targetLang)
}
if err := s.wp.UpdatePostFeaturedMedia(ctx, translatedID, post.FeaturedMedia); err != nil {
return 0, err
}
return translatedID, nil
}
func (s *Service) scan(ctx context.Context) (*models.ImageAuditReport, error) {
start := time.Now()
perPage := 100
page := 1
processed := 0
imagesChecked := 0
var postsWithIssues []models.PostImageAudit
for {
if s.maxPosts > 0 && processed >= s.maxPosts {
break
}
params := wordpress.ListParams{
Status: "publish",
OrderBy: "modified",
Order: "desc",
PerPage: perPage,
Page: page,
}
posts, err := s.wp.ListPosts(ctx, params)
if err != nil {
return nil, err
}
if len(posts) == 0 {
break
}
for _, post := range posts {
if s.maxPosts > 0 && processed >= s.maxPosts {
break
}
processed += 1
content := strings.TrimSpace(post.Content.Rendered)
if content == "" {
continue
}
refs := extractImageRefs(content)
if s.maxImagesPerPost > 0 && len(refs) > s.maxImagesPerPost {
refs = refs[:s.maxImagesPerPost]
}
uniqueRefs := uniqueImageRefs(refs)
imagesChecked += len(uniqueRefs)
issues := s.collectExternalIssues(uniqueRefs)
if len(issues) == 0 {
continue
}
meta := buildMetaScore(post)
postsWithIssues = append(postsWithIssues, models.PostImageAudit{
PostID: post.ID,
PostTitle: post.Title.Rendered,
PostURL: post.Link,
ExternalCount: len(issues),
ExternalImages: issues,
Meta: meta,
})
}
if len(posts) < perPage {
break
}
page += 1
}
sort.Slice(postsWithIssues, func(i, j int) bool {
if postsWithIssues[i].ExternalCount == postsWithIssues[j].ExternalCount {
return postsWithIssues[i].PostID < postsWithIssues[j].PostID
}
return postsWithIssues[i].ExternalCount > postsWithIssues[j].ExternalCount
})
totalExternal := 0
for _, post := range postsWithIssues {
totalExternal += post.ExternalCount
}
mismatches, err := s.findFeaturedMismatches(ctx)
if err != nil {
s.logger.Warn("Failed to check featured image mismatches", "error", err)
}
report := &models.ImageAuditReport{
CreatedAt: time.Now(),
DurationMs: time.Since(start).Milliseconds(),
TotalPosts: processed,
ImagesChecked: imagesChecked,
TotalExternalImages: totalExternal,
Posts: postsWithIssues,
FeaturedImageMismatches: mismatches,
}
return report, nil
}
func (s *Service) findFeaturedMismatches(ctx context.Context) ([]models.FeaturedImageMismatch, error) {
spanishMap := make(map[int]*wordpress.Post)
perPage := 100
page := 1
for {
posts, err := s.wp.ListPosts(ctx, wordpress.ListParams{
Lang: "es",
Status: "publish",
OrderBy: "modified",
Order: "desc",
PerPage: perPage,
Page: page,
})
if err != nil {
return nil, err
}
if len(posts) == 0 {
break
}
for _, post := range posts {
spanishMap[post.ID] = post
}
if len(posts) < perPage {
break
}
page += 1
}
page = 1
var mismatches []models.FeaturedImageMismatch
for {
posts, err := s.wp.ListPosts(ctx, wordpress.ListParams{
Lang: "en",
Status: "publish",
OrderBy: "modified",
Order: "desc",
PerPage: perPage,
Page: page,
})
if err != nil {
return nil, err
}
if len(posts) == 0 {
break
}
for _, post := range posts {
if post.Translations == nil {
continue
}
spanishID := post.Translations["es"]
if spanishID == 0 {
continue
}
spanishPost := spanishMap[spanishID]
if spanishPost == nil {
continue
}
if post.FeaturedMedia == 0 || spanishPost.FeaturedMedia == 0 {
continue
}
if post.FeaturedMedia == spanishPost.FeaturedMedia {
continue
}
mismatches = append(mismatches, models.FeaturedImageMismatch{
EnglishPostID: post.ID,
SpanishPostID: spanishPost.ID,
EnglishTitle: post.Title.Rendered,
SpanishTitle: spanishPost.Title.Rendered,
EnglishURL: post.Link,
SpanishURL: spanishPost.Link,
EnglishFeaturedID: post.FeaturedMedia,
SpanishFeaturedID: spanishPost.FeaturedMedia,
})
}
if len(posts) < perPage {
break
}
page += 1
}
sort.Slice(mismatches, func(i, j int) bool {
return mismatches[i].EnglishPostID < mismatches[j].EnglishPostID
})
return mismatches, nil
}
func (s *Service) collectExternalIssues(refs []imageRef) []models.ExternalImageIssue {
issues := make([]models.ExternalImageIssue, 0)
seen := make(map[string]struct{})
for _, ref := range refs {
resolved, host, ok := s.resolveURL(ref.url)
if !ok {
continue
}
if s.isAllowedHost(host) {
continue
}
if _, exists := seen[resolved]; exists {
continue
}
seen[resolved] = struct{}{}
issues = append(issues, models.ExternalImageIssue{
URL: resolved,
Host: host,
Source: ref.source,
})
}
return issues
}
func (s *Service) externalImageURLs(refs []imageRef) []string {
urls := make([]string, 0)
seen := make(map[string]struct{})
for _, ref := range refs {
resolved, host, ok := s.resolveURL(ref.url)
if !ok {
continue
}
if s.isAllowedHost(host) {
continue
}
if _, exists := seen[resolved]; exists {
continue
}
seen[resolved] = struct{}{}
urls = append(urls, resolved)
}
return urls
}
func (s *Service) resolveURL(raw string) (string, string, bool) {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return "", "", false
}
if strings.HasPrefix(trimmed, "data:") || strings.HasPrefix(trimmed, "blob:") {
return "", "", false
}
parsed, err := url.Parse(trimmed)
if err != nil {
return "", "", false
}
if parsed.Scheme != "" && parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", "", false
}
if parsed.Host == "" {
if s.baseURL == nil {
return "", "", false
}
resolved := s.baseURL.ResolveReference(parsed)
return resolved.String(), resolved.Hostname(), true
}
if parsed.Scheme == "" && s.baseURL != nil && s.baseURL.Scheme != "" {
parsed.Scheme = s.baseURL.Scheme
}
return parsed.String(), parsed.Hostname(), true
}
func (s *Service) isAllowedHost(host string) bool {
trimmed := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(host), "www."))
if trimmed == "" {
return true
}
if s.baseURL != nil {
baseHost := strings.ToLower(strings.TrimPrefix(s.baseURL.Hostname(), "www."))
if baseHost != "" && trimmed == baseHost {
return true
}
}
for _, suffix := range s.allowedSuffixes {
if strings.HasSuffix(trimmed, suffix) {
return true
}
}
return false
}
func extractImageRefs(content string) []imageRef {
root, err := html.Parse(strings.NewReader(content))
if err != nil {
return nil
}
refs := make([]imageRef, 0)
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "img" {
var src string
var srcset string
for _, attr := range n.Attr {
switch strings.ToLower(attr.Key) {
case "src":
src = strings.TrimSpace(attr.Val)
case "srcset":
srcset = strings.TrimSpace(attr.Val)
}
}
if src != "" {
refs = append(refs, imageRef{url: src, source: "src"})
}
if srcset != "" {
refs = append(refs, parseSrcset(srcset)...)
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(root)
return refs
}
func parseSrcset(srcset string) []imageRef {
parts := strings.Split(srcset, ",")
refs := make([]imageRef, 0, len(parts))
for _, part := range parts {
segment := strings.TrimSpace(part)
if segment == "" {
continue
}
fields := strings.Fields(segment)
if len(fields) == 0 {
continue
}
refs = append(refs, imageRef{url: fields[0], source: "srcset"})
}
return refs
}
func uniqueImageRefs(refs []imageRef) []imageRef {
seen := make(map[string]struct{}, len(refs))
unique := make([]imageRef, 0, len(refs))
for _, ref := range refs {
key := ref.url + "|" + ref.source
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
unique = append(unique, ref)
}
return unique
}
func buildMetaScore(post *wordpress.Post) *models.MetaScore {
if post == nil {
return nil
}
seoTitle := getMetaString(post.Meta, wordpress.MetaRankMathTitle, wordpress.MetaYoastTitle)
seoDesc := getMetaString(post.Meta, wordpress.MetaRankMathDescription, wordpress.MetaYoastDescription)
if strings.TrimSpace(seoTitle) == "" && strings.TrimSpace(seoDesc) == "" {
return &models.MetaScore{
Title: seoTitle,
Description: seoDesc,
TitleScore: 0,
DescriptionScore: 0,
TitleIssues: []string{"missing"},
DescriptionIssues: []string{"missing"},
}
}
titleScore, titleIssues := scoreTitle(seoTitle)
descScore, descIssues := scoreDescription(seoDesc)
return &models.MetaScore{
Title: seoTitle,
Description: seoDesc,
TitleScore: titleScore,
DescriptionScore: descScore,
TitleIssues: titleIssues,
DescriptionIssues: descIssues,
}
}
func scoreTitle(title string) (int, []string) {
trimmed := strings.TrimSpace(title)
if trimmed == "" {
return 0, []string{"missing"}
}
length := len([]rune(trimmed))
issues := make([]string, 0)
score := 100
if length < 30 {
score = 40
issues = append(issues, "too_short")
} else if length < 40 {
score = 60
issues = append(issues, "short")
} else if length < 50 {
score = 80
issues = append(issues, "could_be_longer")
} else if length <= 60 {
score = 100
} else if length <= 70 {
score = 80
issues = append(issues, "slightly_long")
} else if length <= 80 {
score = 60
issues = append(issues, "long")
} else {
score = 40
issues = append(issues, "too_long")
}
return score, issues
}
func scoreDescription(desc string) (int, []string) {
trimmed := strings.TrimSpace(desc)
if trimmed == "" {
return 0, []string{"missing"}
}
length := len([]rune(trimmed))
issues := make([]string, 0)
score := 100
if length < 90 {
score = 40
issues = append(issues, "too_short")
} else if length < 120 {
score = 60
issues = append(issues, "short")
} else if length < 150 {
score = 80
issues = append(issues, "could_be_longer")
} else if length <= 160 {
score = 100
} else if length <= 180 {
score = 80
issues = append(issues, "slightly_long")
} else if length <= 200 {
score = 60
issues = append(issues, "long")
} else {
score = 40
issues = append(issues, "too_long")
}
return score, issues
}
func getMetaString(meta map[string]interface{}, keys ...string) string {
if meta == nil {
return ""
}
for _, key := range keys {
if value, ok := meta[key]; ok && value != nil {
if str, ok := value.(string); ok && str != "" {
return str
}
}
}
return ""
}
func (s *Service) convertAndUpload(ctx context.Context, imageURL, title string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, s.downloadTimeout)
defer cancel()
tmpDir, err := os.MkdirTemp("", "wpsk-image-*")
if err != nil {
return "", err
}
defer os.RemoveAll(tmpDir)
inputPath, filename, err := s.downloadImage(ctx, imageURL, tmpDir)
if err != nil {
return "", err
}
baseName := strings.TrimSuffix(filename, filepath.Ext(filename))
if baseName == "" {
baseName = "image"
}
outputName := baseName + ".avif"
outputPath := filepath.Join(tmpDir, outputName)
cmd := exec.CommandContext(ctx, "avifenc", inputPath, outputPath)
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("avifenc failed: %w (%s)", err, strings.TrimSpace(string(output)))
}
_, url, err := s.wp.UploadMedia(ctx, outputPath, outputName, "image/avif", title, "")
if err != nil {
return "", err
}
return url, nil
}
func (s *Service) downloadImage(ctx context.Context, imageURL, dir string) (string, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
if err != nil {
return "", "", err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", "", fmt.Errorf("download failed: status %d", resp.StatusCode)
}
filename := extractFilename(imageURL)
if filename == "" {
filename = "image"
}
inputPath := filepath.Join(dir, filename)
file, err := os.Create(inputPath)
if err != nil {
return "", "", err
}
defer file.Close()
limited := &io.LimitedReader{R: resp.Body, N: s.maxDownloadBytes}
written, err := io.Copy(file, limited)
if err != nil {
return "", "", err
}
if written >= s.maxDownloadBytes {
return "", "", errors.New("download exceeded max bytes")
}
return inputPath, filename, nil
}
func extractFilename(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil {
return ""
}
base := path.Base(parsed.Path)
return strings.TrimSpace(base)
}
+109 -3
View File
@@ -7,7 +7,9 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"seo-optimizer/internal/logger"
@@ -36,6 +38,13 @@ type Client struct {
userAgent string
http *http.Client
logger *logger.Logger
mu sync.Mutex
lastReq time.Time
minGap time.Duration
cacheTTL time.Duration
cacheKey string
cacheAt time.Time
cacheData []Post
}
// NewClient creates a Reddit client with sane defaults.
@@ -53,7 +62,9 @@ func NewClient(baseURL string, log *logger.Logger) *Client {
http: &http.Client{
Timeout: 20 * time.Second,
},
logger: log.WithComponent("reddit"),
logger: log.WithComponent("reddit"),
minGap: 5 * time.Second,
cacheTTL: 10 * time.Minute,
}
}
@@ -70,6 +81,16 @@ func (c *Client) FetchPosts(ctx context.Context, subreddit, listing, timeRange s
listing = "top"
}
cacheKey := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%s|%s|%s|%d", subreddit, listing, timeRange, limit)))
if cached, ok := c.getCached(cacheKey); ok {
c.logger.Debug("Using cached Reddit response", "subreddit", subreddit, "listing", listing)
return cached, nil
}
if err := c.waitForGap(ctx); err != nil {
return nil, err
}
path := fmt.Sprintf("/r/%s/%s.json", url.PathEscape(subreddit), listing)
params := url.Values{}
params.Set("limit", fmt.Sprintf("%d", limit))
@@ -118,7 +139,7 @@ func (c *Client) FetchPosts(ctx context.Context, subreddit, listing, timeRange s
Author: data.Author,
Score: data.Score,
NumComments: data.NumComments,
CreatedUTC: data.CreatedUTC,
CreatedUTC: int64(data.CreatedUTC),
Subreddit: data.Subreddit,
IsSelf: data.IsSelf,
})
@@ -130,9 +151,52 @@ func (c *Client) FetchPosts(ctx context.Context, subreddit, listing, timeRange s
"count", len(posts),
)
c.setCache(cacheKey, posts)
return posts, nil
}
func (c *Client) waitForGap(ctx context.Context) error {
c.mu.Lock()
gap := c.minGap - time.Since(c.lastReq)
if gap > 0 {
c.mu.Unlock()
timer := time.NewTimer(gap)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
}
c.mu.Lock()
}
c.lastReq = time.Now()
c.mu.Unlock()
return nil
}
func (c *Client) getCached(key string) ([]Post, bool) {
c.mu.Lock()
defer c.mu.Unlock()
if c.cacheKey != key {
return nil, false
}
if time.Since(c.cacheAt) > c.cacheTTL {
return nil, false
}
dup := make([]Post, len(c.cacheData))
copy(dup, c.cacheData)
return dup, true
}
func (c *Client) setCache(key string, posts []Post) {
c.mu.Lock()
defer c.mu.Unlock()
c.cacheKey = key
c.cacheAt = time.Now()
c.cacheData = make([]Post, len(posts))
copy(c.cacheData, posts)
}
type redditListing struct {
Data struct {
Children []struct {
@@ -144,7 +208,7 @@ type redditListing struct {
Author string `json:"author"`
Score int `json:"score"`
NumComments int `json:"num_comments"`
CreatedUTC int64 `json:"created_utc"`
CreatedUTC unixSeconds
Subreddit string `json:"subreddit"`
IsSelf bool `json:"is_self"`
Stickied bool `json:"stickied"`
@@ -153,3 +217,45 @@ type redditListing struct {
} `json:"children"`
} `json:"data"`
}
type unixSeconds int64
func (u *unixSeconds) UnmarshalJSON(data []byte) error {
var num json.Number
if err := json.Unmarshal(data, &num); err == nil {
if strings.Contains(num.String(), ".") {
f, err := num.Float64()
if err != nil {
return fmt.Errorf("invalid unix seconds float: %w", err)
}
*u = unixSeconds(int64(f))
return nil
}
i, err := num.Int64()
if err != nil {
return fmt.Errorf("invalid unix seconds int: %w", err)
}
*u = unixSeconds(i)
return nil
}
var text string
if err := json.Unmarshal(data, &text); err == nil {
if strings.Contains(text, ".") {
f, err := strconv.ParseFloat(text, 64)
if err != nil {
return fmt.Errorf("invalid unix seconds string float: %w", err)
}
*u = unixSeconds(int64(f))
return nil
}
i, err := strconv.ParseInt(text, 10, 64)
if err != nil {
return fmt.Errorf("invalid unix seconds string int: %w", err)
}
*u = unixSeconds(i)
return nil
}
return fmt.Errorf("invalid unix seconds payload: %s", string(data))
}
+101
View File
@@ -0,0 +1,101 @@
package scheduler
import (
"context"
"fmt"
"time"
"seo-optimizer/internal/brokenlinks"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/storage"
)
// BrokenLinkScheduler handles periodic broken link scans.
type BrokenLinkScheduler struct {
service *brokenlinks.Service
interval time.Duration
stopCh chan struct{}
logger *logger.Logger
audit *storage.AuditStorage
}
func NewBrokenLinkScheduler(service *brokenlinks.Service, intervalHours float64, audit *storage.AuditStorage, log *logger.Logger) *BrokenLinkScheduler {
if log == nil {
log = logger.NewDefaultLogger()
}
if intervalHours <= 0 {
intervalHours = 168
}
return &BrokenLinkScheduler{
service: service,
interval: time.Duration(intervalHours * float64(time.Hour)),
stopCh: make(chan struct{}),
logger: log.WithComponent("broken_link_scheduler"),
audit: audit,
}
}
// Start begins the scheduler loop.
func (s *BrokenLinkScheduler) Start(ctx context.Context) error {
s.logger.Info("Broken link scheduler started", "interval", s.interval)
if err := s.runOnce(ctx); err != nil {
s.logger.Error("Initial broken link scan failed", "error", err)
}
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err := s.runOnce(ctx); err != nil {
s.logger.Error("Broken link scan failed", "error", err)
}
case <-ctx.Done():
s.logger.Info("Broken link scheduler stopped by context")
return ctx.Err()
case <-s.stopCh:
s.logger.Info("Broken link scheduler stopped by signal")
return nil
}
}
}
// Stop stops the scheduler.
func (s *BrokenLinkScheduler) Stop() {
close(s.stopCh)
}
func (s *BrokenLinkScheduler) runOnce(ctx context.Context) error {
if s.service == nil {
return fmt.Errorf("broken link service is nil")
}
start := time.Now()
report, err := s.service.RunScan(ctx)
if err != nil {
s.recordAudit("broken_links_scan_failed", fmt.Sprintf("Broken link scan failed: %v", err))
return err
}
s.logger.Info("Broken link scan completed",
"posts_checked", report.TotalPosts,
"links_checked", report.LinksChecked,
"broken_links", report.TotalBrokenLinks,
"duration", time.Since(start),
)
s.recordAudit("broken_links_scanned", fmt.Sprintf("Scanned %d posts (%d broken links)", report.TotalPosts, report.TotalBrokenLinks))
return nil
}
func (s *BrokenLinkScheduler) recordAudit(action, summary string) {
if s.audit == nil {
return
}
_ = s.audit.Save(&storage.AuditRecord{
Actor: "scheduler",
Action: action,
Summary: summary,
})
}
+24 -1
View File
@@ -2,10 +2,12 @@ package scheduler
import (
"context"
"fmt"
"time"
"seo-optimizer/internal/ideas"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/storage"
)
// IdeaScheduler handles periodic idea generation.
@@ -16,15 +18,20 @@ type IdeaScheduler struct {
interval time.Duration
stopCh chan struct{}
logger *logger.Logger
audit *storage.AuditStorage
}
func NewIdeaScheduler(service *ideas.Service, intervalHours float64, enableIdeas bool, enableOutreach bool, log *logger.Logger) *IdeaScheduler {
func NewIdeaScheduler(service *ideas.Service, intervalHours float64, enableIdeas bool, enableOutreach bool, audit *storage.AuditStorage, log *logger.Logger) *IdeaScheduler {
if log == nil {
log = logger.NewDefaultLogger()
}
if intervalHours <= 0 {
intervalHours = 24
}
if intervalHours < 12 {
log.Warn("Idea scheduler interval too low; clamping to 12h to reduce Reddit load", "interval_hours", intervalHours)
intervalHours = 12
}
return &IdeaScheduler{
service: service,
@@ -33,6 +40,7 @@ func NewIdeaScheduler(service *ideas.Service, intervalHours float64, enableIdeas
interval: time.Duration(intervalHours * float64(time.Hour)),
stopCh: make(chan struct{}),
logger: log.WithComponent("idea_scheduler"),
audit: audit,
}
}
@@ -73,6 +81,7 @@ func (s *IdeaScheduler) runOnce(ctx context.Context) error {
if s.enableIdeas {
ideas, err := s.service.GenerateDailyIdeas(ctx)
if err != nil {
s.recordAudit("ideas_generation_failed", fmt.Sprintf("Idea generation failed: %v", err))
return err
}
@@ -80,18 +89,32 @@ func (s *IdeaScheduler) runOnce(ctx context.Context) error {
"count", len(ideas),
"duration", time.Since(start),
)
s.recordAudit("ideas_generated", fmt.Sprintf("Scheduler generated %d ideas", len(ideas)))
}
if s.enableOutreach {
outreach, err := s.service.GenerateOutreachSuggestions(ctx)
if err != nil {
s.logger.Warn("Outreach generation failed", "error", err)
s.recordAudit("outreach_generation_failed", fmt.Sprintf("Outreach generation failed: %v", err))
} else {
s.logger.Info("Outreach generation completed",
"count", len(outreach),
"duration", time.Since(start),
)
s.recordAudit("outreach_generated", fmt.Sprintf("Scheduler generated %d outreach suggestions", len(outreach)))
}
}
return nil
}
func (s *IdeaScheduler) recordAudit(action, summary string) {
if s.audit == nil {
return
}
_ = s.audit.Save(&storage.AuditRecord{
Actor: "scheduler",
Action: action,
Summary: summary,
})
}
+101
View File
@@ -0,0 +1,101 @@
package scheduler
import (
"context"
"fmt"
"time"
"seo-optimizer/internal/imageaudit"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/storage"
)
// ImageAuditScheduler handles periodic image audits.
type ImageAuditScheduler struct {
service *imageaudit.Service
interval time.Duration
stopCh chan struct{}
logger *logger.Logger
audit *storage.AuditStorage
}
func NewImageAuditScheduler(service *imageaudit.Service, intervalHours float64, audit *storage.AuditStorage, log *logger.Logger) *ImageAuditScheduler {
if log == nil {
log = logger.NewDefaultLogger()
}
if intervalHours <= 0 {
intervalHours = 168
}
return &ImageAuditScheduler{
service: service,
interval: time.Duration(intervalHours * float64(time.Hour)),
stopCh: make(chan struct{}),
logger: log.WithComponent("image_audit_scheduler"),
audit: audit,
}
}
// Start begins the scheduler loop.
func (s *ImageAuditScheduler) Start(ctx context.Context) error {
s.logger.Info("Image audit scheduler started", "interval", s.interval)
if err := s.runOnce(ctx); err != nil {
s.logger.Error("Initial image audit failed", "error", err)
}
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err := s.runOnce(ctx); err != nil {
s.logger.Error("Image audit failed", "error", err)
}
case <-ctx.Done():
s.logger.Info("Image audit scheduler stopped by context")
return ctx.Err()
case <-s.stopCh:
s.logger.Info("Image audit scheduler stopped by signal")
return nil
}
}
}
// Stop stops the scheduler.
func (s *ImageAuditScheduler) Stop() {
close(s.stopCh)
}
func (s *ImageAuditScheduler) runOnce(ctx context.Context) error {
if s.service == nil {
return fmt.Errorf("image audit service is nil")
}
start := time.Now()
report, err := s.service.RunAudit(ctx)
if err != nil {
s.recordAudit("image_audit_failed", fmt.Sprintf("Image audit failed: %v", err))
return err
}
s.logger.Info("Image audit completed",
"posts_checked", report.TotalPosts,
"images_checked", report.ImagesChecked,
"external_images", report.TotalExternalImages,
"duration", time.Since(start),
)
s.recordAudit("image_audit_completed", fmt.Sprintf("Image audit completed (%d external images)", report.TotalExternalImages))
return nil
}
func (s *ImageAuditScheduler) recordAudit(action, summary string) {
if s.audit == nil {
return
}
_ = s.audit.Save(&storage.AuditRecord{
Actor: "scheduler",
Action: action,
Summary: summary,
})
}
+18 -1
View File
@@ -2,10 +2,12 @@ package scheduler
import (
"context"
"fmt"
"time"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/newsletter"
"seo-optimizer/internal/storage"
)
// NewsletterScheduler handles periodic newsletter generation.
@@ -14,9 +16,10 @@ type NewsletterScheduler struct {
interval time.Duration
stopCh chan struct{}
logger *logger.Logger
audit *storage.AuditStorage
}
func NewNewsletterScheduler(service *newsletter.Service, intervalHours float64, log *logger.Logger) *NewsletterScheduler {
func NewNewsletterScheduler(service *newsletter.Service, intervalHours float64, audit *storage.AuditStorage, log *logger.Logger) *NewsletterScheduler {
if log == nil {
log = logger.NewDefaultLogger()
}
@@ -29,6 +32,7 @@ func NewNewsletterScheduler(service *newsletter.Service, intervalHours float64,
interval: time.Duration(intervalHours * float64(time.Hour)),
stopCh: make(chan struct{}),
logger: log.WithComponent("newsletter_scheduler"),
audit: audit,
}
}
@@ -68,6 +72,7 @@ func (s *NewsletterScheduler) runOnce(ctx context.Context) error {
start := time.Now()
draft, err := s.service.GenerateWeeklyDraft(ctx)
if err != nil {
s.recordAudit("newsletter_generation_failed", fmt.Sprintf("Newsletter generation failed: %v", err))
return err
}
@@ -75,5 +80,17 @@ func (s *NewsletterScheduler) runOnce(ctx context.Context) error {
"id", draft.ID,
"duration", time.Since(start),
)
s.recordAudit("newsletter_generated", fmt.Sprintf("Scheduler generated newsletter draft %s", draft.ID))
return nil
}
func (s *NewsletterScheduler) recordAudit(action, summary string) {
if s.audit == nil {
return
}
_ = s.audit.Save(&storage.AuditRecord{
Actor: "scheduler",
Action: action,
Summary: summary,
})
}
+108 -14
View File
@@ -6,6 +6,7 @@ import (
"regexp"
"strconv"
"strings"
"unicode"
"seo-optimizer/internal/logger"
"seo-optimizer/pkg/models"
@@ -118,20 +119,9 @@ func (cb *ContentBuilder) applyChange(content string, change models.ContentChang
return strings.Replace(content, change.Original, change.Updated, 1)
}
// Try normalized match (handle HTML entities, extra whitespace)
normalizedOriginal := normalizeText(change.Original)
normalizedContent := normalizeText(content)
if strings.Contains(normalizedContent, normalizedOriginal) {
// Find position in normalized content
idx := strings.Index(normalizedContent, normalizedOriginal)
if idx >= 0 {
// Find corresponding position in original content
// This is approximate but should work for most cases
before := content[:idx]
after := content[idx+len(change.Original):]
return before + change.Updated + after
}
// Try normalized match (handle quotes and whitespace)
if start, end, ok := findNormalizedSpan(content, change.Original); ok {
return content[:start] + change.Updated + content[end:]
}
cb.logger.Warn("Could not find exact match for content change",
@@ -149,6 +139,10 @@ func (cb *ContentBuilder) applyFactCheckUpdate(content string, factCheck models.
return strings.Replace(content, factCheck.Claim, factCheck.UpdatedText, 1)
}
if start, end, ok := findNormalizedSpan(content, factCheck.Claim); ok {
return content[:start] + factCheck.UpdatedText + content[end:]
}
cb.logger.Warn("Could not find fact check claim in content",
"claim", truncate(factCheck.Claim, 100),
)
@@ -201,11 +195,24 @@ func (cb *ContentBuilder) insertInternalLink(content string, link models.Interna
}
}
if _, end, ok := findNormalizedSpan(content, link.InsertionContext); ok {
remainder := content[end:]
endIdx := findSentenceEnd(remainder)
if endIdx > 0 {
return content[:end] + remainder[:endIdx] + " " + linkHTML + remainder[endIdx:]
}
return content[:end] + " " + linkHTML + content[end:]
}
// Fallback: try to insert the link by replacing the anchor text if it appears in content
if strings.Contains(content, link.AnchorText) && !cb.isInsideTag(content, link.AnchorText) {
return strings.Replace(content, link.AnchorText, linkHTML, 1)
}
if updated, ok := insertAfterFirstParagraph(content, linkHTML); ok {
return updated
}
cb.logger.Warn("Could not find appropriate insertion point for internal link",
"anchor", link.AnchorText,
"context", truncate(link.InsertionContext, 100),
@@ -230,6 +237,93 @@ func (cb *ContentBuilder) isInsideTag(content, text string) bool {
return lastOpen > lastClose
}
func normalizeForMatch(input string) string {
normalized, _ := normalizeWithMap(input)
return normalized
}
func findNormalizedSpan(content, needle string) (int, int, bool) {
if strings.TrimSpace(needle) == "" {
return 0, 0, false
}
normalizedContent, indexMap := normalizeWithMap(content)
normalizedNeedle := normalizeForMatch(needle)
if normalizedNeedle == "" {
return 0, 0, false
}
idx := strings.Index(normalizedContent, normalizedNeedle)
if idx == -1 {
return 0, 0, false
}
endIdx := idx + len(normalizedNeedle) - 1
if idx < 0 || endIdx >= len(indexMap) {
return 0, 0, false
}
start := indexMap[idx]
end := indexMap[endIdx] + 1
if start < 0 || end > len(content) || start >= end {
return 0, 0, false
}
return start, end, true
}
func normalizeWithMap(input string) (string, []int) {
var b strings.Builder
indexMap := make([]int, 0, len(input))
lastSpace := false
for i, r := range input {
if unicode.IsSpace(r) {
if lastSpace {
continue
}
lastSpace = true
b.WriteByte(' ')
indexMap = append(indexMap, i)
continue
}
lastSpace = false
r = normalizeQuote(r)
token := strings.ToLower(string(r))
b.WriteString(token)
for j := 0; j < len(token); j++ {
indexMap = append(indexMap, i)
}
}
return b.String(), indexMap
}
func normalizeQuote(r rune) rune {
switch r {
case '“', '”':
return '"'
case '', '':
return '\''
default:
return r
}
}
func insertAfterFirstParagraph(content, linkHTML string) (string, bool) {
if strings.TrimSpace(content) == "" {
return content, false
}
if idx := strings.Index(content, "</p>"); idx != -1 {
insertAt := idx + len("</p>")
return content[:insertAt] + " " + linkHTML + content[insertAt:], true
}
if idx := strings.Index(content, "\n\n"); idx != -1 {
return content[:idx] + "\n\n" + linkHTML + "\n\n" + content[idx+2:], true
}
return content + "\n\n" + linkHTML, true
}
// findSentenceEnd finds the end of the current sentence (. ! ?)
func findSentenceEnd(text string) int {
for i, char := range text {
+18 -3
View File
@@ -168,7 +168,12 @@ func (o *Optimizer) optimizePost(ctx context.Context, post *wordpress.Post, lang
if checkCooldown && o.wasRecentlyOptimized(post, o.config.Optimization.CooldownDays) {
reason := "post optimized within cooldown window"
o.logger.Info("Skipping optimization", "post_id", post.ID, "reason", reason)
o.logger.Info("Skipping optimization",
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
"reason", reason,
)
if err := o.saveSkippedRecord(post, effectiveLanguage, "recently_optimized", nil); err != nil {
o.logger.Warn("Failed to save skipped optimization record", "error", err)
}
@@ -212,7 +217,12 @@ func (o *Optimizer) optimizePost(ctx context.Context, post *wordpress.Post, lang
}
// 3. Call LLM for optimization
o.logger.Info("Calling LLM for optimization", "provider", o.llmProvider)
o.logger.Info("Calling LLM for optimization",
"provider", o.llmProvider,
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
)
optimization, err := o.llmClient.OptimizePost(ctx, post, internalPosts, categoryMap, tagMap)
if err != nil {
@@ -222,7 +232,12 @@ func (o *Optimizer) optimizePost(ctx context.Context, post *wordpress.Post, lang
// Skip if the post is too recent and no modifications were required
if o.shouldSkipRecentNoChanges(post, optimization) {
reason := "post too recent and no changes required"
o.logger.Info("Skipping optimization", "post_id", post.ID, "reason", reason)
o.logger.Info("Skipping optimization",
"post_id", post.ID,
"title", post.Title.Rendered,
"slug", post.Slug,
"reason", reason,
)
if err := o.saveSkippedRecord(post, effectiveLanguage, "no_changes_recent", optimization); err != nil {
o.logger.Warn("Failed to save skipped optimization record", "error", err)
}
+128
View File
@@ -0,0 +1,128 @@
package storage
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"time"
"seo-optimizer/internal/logger"
"seo-optimizer/pkg/models"
)
// BrokenLinkStorage manages file-based storage of broken link reports.
type BrokenLinkStorage struct {
basePath string
logger *logger.Logger
}
// NewBrokenLinkStorage creates a new broken link report storage instance.
func NewBrokenLinkStorage(basePath string, log *logger.Logger) *BrokenLinkStorage {
if log == nil {
log = logger.NewDefaultLogger()
}
return &BrokenLinkStorage{
basePath: basePath,
logger: log.WithComponent("broken_link_storage"),
}
}
// Initialize creates the storage directory.
func (s *BrokenLinkStorage) Initialize() error {
if err := os.MkdirAll(s.basePath, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", s.basePath, err)
}
return nil
}
// Save stores a broken link report.
func (s *BrokenLinkStorage) Save(report *models.BrokenLinkReport) error {
if report == nil {
return fmt.Errorf("report is nil")
}
if report.CreatedAt.IsZero() {
report.CreatedAt = time.Now()
}
if report.ID == "" {
report.ID = fmt.Sprintf("%d", report.CreatedAt.Unix())
}
filename := fmt.Sprintf("%s_%d.json", report.ID, report.CreatedAt.Unix())
path := filepath.Join(s.basePath, filename)
data, err := json.MarshalIndent(report, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal report: %w", err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
return fmt.Errorf("failed to write report: %w", err)
}
s.logger.Debug("Broken link report saved", "file", path)
return nil
}
// List returns reports sorted by newest first. Limit <= 0 returns all.
func (s *BrokenLinkStorage) List(limit int) ([]*models.BrokenLinkReport, error) {
files, err := os.ReadDir(s.basePath)
if err != nil {
if os.IsNotExist(err) {
return []*models.BrokenLinkReport{}, nil
}
return nil, fmt.Errorf("failed to read directory: %w", err)
}
var reports []*models.BrokenLinkReport
for _, file := range files {
if file.IsDir() || filepath.Ext(file.Name()) != ".json" {
continue
}
path := filepath.Join(s.basePath, file.Name())
report, err := s.readReport(path)
if err != nil {
s.logger.Warn("Failed to read broken link report", "file", path, "error", err)
continue
}
reports = append(reports, report)
}
sort.Slice(reports, func(i, j int) bool {
return reports[i].CreatedAt.After(reports[j].CreatedAt)
})
if limit > 0 && len(reports) > limit {
reports = reports[:limit]
}
return reports, nil
}
// Latest returns the most recent report.
func (s *BrokenLinkStorage) Latest() (*models.BrokenLinkReport, error) {
reports, err := s.List(1)
if err != nil {
return nil, err
}
if len(reports) == 0 {
return nil, nil
}
return reports[0], nil
}
func (s *BrokenLinkStorage) readReport(path string) (*models.BrokenLinkReport, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var report models.BrokenLinkReport
if err := json.Unmarshal(data, &report); err != nil {
return nil, err
}
return &report, nil
}
+128
View File
@@ -0,0 +1,128 @@
package storage
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"time"
"seo-optimizer/internal/logger"
"seo-optimizer/pkg/models"
)
// ImageAuditStorage manages file-based storage of image audit reports.
type ImageAuditStorage struct {
basePath string
logger *logger.Logger
}
// NewImageAuditStorage creates a new image audit storage instance.
func NewImageAuditStorage(basePath string, log *logger.Logger) *ImageAuditStorage {
if log == nil {
log = logger.NewDefaultLogger()
}
return &ImageAuditStorage{
basePath: basePath,
logger: log.WithComponent("image_audit_storage"),
}
}
// Initialize creates the storage directory.
func (s *ImageAuditStorage) Initialize() error {
if err := os.MkdirAll(s.basePath, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", s.basePath, err)
}
return nil
}
// Save stores an image audit report.
func (s *ImageAuditStorage) Save(report *models.ImageAuditReport) error {
if report == nil {
return fmt.Errorf("report is nil")
}
if report.CreatedAt.IsZero() {
report.CreatedAt = time.Now()
}
if report.ID == "" {
report.ID = fmt.Sprintf("%d", report.CreatedAt.Unix())
}
filename := fmt.Sprintf("%s_%d.json", report.ID, report.CreatedAt.Unix())
path := filepath.Join(s.basePath, filename)
data, err := json.MarshalIndent(report, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal report: %w", err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
return fmt.Errorf("failed to write report: %w", err)
}
s.logger.Debug("Image audit report saved", "file", path)
return nil
}
// List returns reports sorted by newest first. Limit <= 0 returns all.
func (s *ImageAuditStorage) List(limit int) ([]*models.ImageAuditReport, error) {
files, err := os.ReadDir(s.basePath)
if err != nil {
if os.IsNotExist(err) {
return []*models.ImageAuditReport{}, nil
}
return nil, fmt.Errorf("failed to read directory: %w", err)
}
var reports []*models.ImageAuditReport
for _, file := range files {
if file.IsDir() || filepath.Ext(file.Name()) != ".json" {
continue
}
path := filepath.Join(s.basePath, file.Name())
report, err := s.readReport(path)
if err != nil {
s.logger.Warn("Failed to read image audit report", "file", path, "error", err)
continue
}
reports = append(reports, report)
}
sort.Slice(reports, func(i, j int) bool {
return reports[i].CreatedAt.After(reports[j].CreatedAt)
})
if limit > 0 && len(reports) > limit {
reports = reports[:limit]
}
return reports, nil
}
// Latest returns the most recent report.
func (s *ImageAuditStorage) Latest() (*models.ImageAuditReport, error) {
reports, err := s.List(1)
if err != nil {
return nil, err
}
if len(reports) == 0 {
return nil, nil
}
return reports[0], nil
}
func (s *ImageAuditStorage) readReport(path string) (*models.ImageAuditReport, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var report models.ImageAuditReport
if err := json.Unmarshal(data, &report); err != nil {
return nil, err
}
return &report, nil
}
+194
View File
@@ -0,0 +1,194 @@
package translation
import (
"context"
"fmt"
"strings"
"time"
"seo-optimizer/internal/agent"
"seo-optimizer/internal/logger"
"seo-optimizer/internal/wordpress"
"seo-optimizer/pkg/models"
)
type Service struct {
wp *wordpress.Client
client *agent.DeepSeekClient
logger *logger.Logger
baseURL string
}
func NewService(wp *wordpress.Client, client *agent.DeepSeekClient, log *logger.Logger) *Service {
if log == nil {
log = logger.NewDefaultLogger()
}
baseURL := ""
if wp != nil {
baseURL = wp.BaseURL()
}
return &Service{
wp: wp,
client: client,
logger: log.WithComponent("translation"),
baseURL: baseURL,
}
}
// ListOrphanPosts finds posts missing the target language translation.
func (s *Service) ListOrphanPosts(ctx context.Context, sourceLang, targetLang string) ([]models.OrphanPost, error) {
if sourceLang == "" {
sourceLang = "en"
}
if targetLang == "" {
targetLang = "es"
}
perPage := 100
page := 1
var orphans []models.OrphanPost
for {
posts, err := s.wp.ListPosts(ctx, wordpress.ListParams{
Lang: sourceLang,
Status: "publish",
OrderBy: "modified",
Order: "desc",
PerPage: perPage,
Page: page,
})
if err != nil {
return nil, err
}
if len(posts) == 0 {
break
}
for _, post := range posts {
if post == nil {
continue
}
if post.Translations != nil {
if id := post.Translations[targetLang]; id != 0 {
continue
}
}
orphans = append(orphans, models.OrphanPost{
PostID: post.ID,
Title: post.Title.Rendered,
URL: post.Link,
Language: sourceLang,
MissingLang: targetLang,
ModifiedAt: post.Modified.Time,
})
}
if len(posts) < perPage {
break
}
page += 1
}
return orphans, nil
}
// TranslatePost creates a Spanish draft linked via Polylang.
func (s *Service) TranslatePost(ctx context.Context, postID int, targetLang string) (int, error) {
if s.client == nil {
return 0, fmt.Errorf("deepseek client not configured")
}
if targetLang == "" {
targetLang = "es"
}
post, err := s.wp.GetPostForEdit(ctx, postID, "")
if err != nil {
return 0, err
}
seoTitle := getMetaString(post.Meta, wordpress.MetaRankMathTitle, wordpress.MetaYoastTitle)
seoDesc := getMetaString(post.Meta, wordpress.MetaRankMathDescription, wordpress.MetaYoastDescription)
content := strings.TrimSpace(post.Content.Raw)
if content == "" {
content = post.Content.Rendered
}
promptData := agent.PromptData{
Language: "Spanish",
PostID: post.ID,
Title: post.Title.Rendered,
SEOTitle: seoTitle,
MetaDescription: seoDesc,
Slug: post.Slug,
Content: content,
LastModified: post.Modified.Format(time.RFC3339),
}
prompt, err := agent.RenderTranslationPrompt(promptData)
if err != nil {
return 0, err
}
draft, err := s.client.TranslatePost(ctx, prompt)
if err != nil {
return 0, err
}
meta := map[string]interface{}{
wordpress.MetaRankMathTitle: draft.SEOTitle,
wordpress.MetaRankMathDescription: draft.MetaDescription,
wordpress.MetaYoastTitle: draft.SEOTitle,
wordpress.MetaYoastDescription: draft.MetaDescription,
}
create := wordpress.PostCreate{
Status: "draft",
Title: draft.Title,
Content: draft.ContentHTML,
Excerpt: draft.Excerpt,
Lang: targetLang,
Meta: meta,
Slug: strings.TrimSpace(draft.Slug),
}
newPostID, err := s.wp.CreateDraftPost(ctx, create)
if err != nil {
return 0, err
}
translations := map[string]int{}
if post.Translations != nil {
for lang, id := range post.Translations {
if id != 0 {
translations[lang] = id
}
}
}
originalLang := post.Lang
if strings.TrimSpace(originalLang) == "" {
originalLang = "en"
}
translations[originalLang] = post.ID
translations[targetLang] = newPostID
if err := s.wp.UpdatePostTranslations(ctx, post.ID, translations); err != nil {
s.logger.Warn("Failed to update translations on original post", "error", err)
}
if err := s.wp.UpdatePostTranslations(ctx, newPostID, translations); err != nil {
s.logger.Warn("Failed to update translations on translated post", "error", err)
}
return newPostID, nil
}
func getMetaString(meta map[string]interface{}, keys ...string) string {
if meta == nil {
return ""
}
for _, key := range keys {
if value, ok := meta[key]; ok && value != nil {
if str, ok := value.(string); ok && str != "" {
return str
}
}
}
return ""
}
+109
View File
@@ -7,8 +7,11 @@ import (
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@@ -51,6 +54,11 @@ func NewClient(baseURL, username, appPassword string, timeout time.Duration, log
}
}
// BaseURL returns the configured WordPress base URL.
func (c *Client) BaseURL() string {
return c.baseURL
}
// GetPost retrieves a single post by ID
func (c *Client) GetPost(ctx context.Context, postID int, lang string) (*Post, error) {
endpoint := fmt.Sprintf("/wp-json/wp/v2/posts/%d", postID)
@@ -173,6 +181,93 @@ func (c *Client) UpdatePostMeta(ctx context.Context, postID int, metaKey string,
return nil
}
// UpdatePostFeaturedMedia updates the featured image for a post.
func (c *Client) UpdatePostFeaturedMedia(ctx context.Context, postID int, featuredMedia int) error {
endpoint := fmt.Sprintf("/wp-json/wp/v2/posts/%d", postID)
update := map[string]interface{}{
"featured_media": featuredMedia,
}
if err := c.post(ctx, endpoint, update, nil); err != nil {
return fmt.Errorf("failed to update featured media: %w", err)
}
return nil
}
// UpdatePostTranslations updates the Polylang translations mapping for a post.
func (c *Client) UpdatePostTranslations(ctx context.Context, postID int, translations map[string]int) error {
endpoint := fmt.Sprintf("/wp-json/wp/v2/posts/%d", postID)
update := map[string]interface{}{
"translations": translations,
}
if err := c.post(ctx, endpoint, update, nil); err != nil {
return fmt.Errorf("failed to update post translations: %w", err)
}
return nil
}
// UploadMedia uploads a media file to WordPress and returns attachment ID and URL.
func (c *Client) UploadMedia(ctx context.Context, filePath, filename, mimeType, title, altText string) (int, string, error) {
file, err := os.Open(filePath)
if err != nil {
return 0, "", err
}
defer file.Close()
if filename == "" {
filename = filepath.Base(filePath)
}
if mimeType == "" {
mimeType = "application/octet-stream"
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", filename)
if err != nil {
return 0, "", err
}
if _, err := io.Copy(part, file); err != nil {
return 0, "", err
}
if title != "" {
_ = writer.WriteField("title", title)
}
if altText != "" {
_ = writer.WriteField("alt_text", altText)
}
if err := writer.Close(); err != nil {
return 0, "", err
}
endpoint := c.baseURL + "/wp-json/wp/v2/media"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, body)
if err != nil {
return 0, "", err
}
req.Header.Set("Authorization", c.authHeader)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("Accept", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return 0, "", err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
b, _ := io.ReadAll(resp.Body)
return 0, "", fmt.Errorf("media upload failed: status %d, body: %s", resp.StatusCode, string(b))
}
var payload struct {
ID int `json:"id"`
SourceURL string `json:"source_url"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
return 0, "", err
}
return payload.ID, payload.SourceURL, nil
}
// CreateDraftRevision creates a linked draft copy of an existing post
// This allows reviewing changes without unpublishing the original
// The draft is linked via meta field _draft_of_post_id
@@ -205,6 +300,8 @@ func (c *Client) CreateDraftRevision(ctx context.Context, postID int, updates Po
c.logger.Info("Creating linked draft copy",
"original_post_id", postID,
"original_title", originalPost.Title.Rendered,
"original_slug", originalPost.Slug,
"has_title", updates.Title != "",
"has_content", updates.Content != "",
"has_excerpt", updates.Excerpt != "",
@@ -218,6 +315,8 @@ func (c *Client) CreateDraftRevision(ctx context.Context, postID int, updates Po
c.logger.Info("Draft copy created successfully",
"draft_post_id", result.ID,
"draft_title", result.Title.Rendered,
"draft_slug", result.Slug,
"original_post_id", postID,
)
@@ -263,6 +362,8 @@ func (c *Client) UpdatePostStatus(ctx context.Context, postID int, status string
c.logger.Info("Post status updated",
"post_id", postID,
"title", result.Title.Rendered,
"slug", result.Slug,
"status", status,
)
@@ -301,6 +402,8 @@ func (c *Client) ApplyDraftToOriginal(ctx context.Context, draftPostID int) erro
c.logger.Info("Applying draft to original post",
"draft_post_id", draftPostID,
"draft_title", draftPost.Title.Rendered,
"draft_slug", draftPost.Slug,
"original_post_id", originalPostID,
)
@@ -351,6 +454,8 @@ func (c *Client) ApplyDraftToOriginal(ctx context.Context, draftPostID int) erro
c.logger.Info("Draft post deleted successfully",
"draft_post_id", draftPostID,
"title", draftPost.Title.Rendered,
"slug", draftPost.Slug,
)
return nil
@@ -370,6 +475,8 @@ func (c *Client) ApplyDraftToOriginalWithID(ctx context.Context, draftPostID, or
c.logger.Info("Applying draft to original post (explicit original ID)",
"draft_post_id", draftPostID,
"draft_title", draftPost.Title.Rendered,
"draft_slug", draftPost.Slug,
"original_post_id", originalPostID,
)
@@ -438,6 +545,8 @@ func (c *Client) DeleteDraft(ctx context.Context, draftPostID int) error {
c.logger.Info("Draft post deleted successfully",
"draft_post_id", draftPostID,
"title", draftPost.Title.Rendered,
"slug", draftPost.Slug,
)
return nil
+1
View File
@@ -114,6 +114,7 @@ type PostUpdate struct {
type PostCreate struct {
Status string `json:"status"`
Title string `json:"title,omitempty"`
Slug string `json:"slug,omitempty"`
Content string `json:"content,omitempty"`
Excerpt string `json:"excerpt,omitempty"`
Lang string `json:"lang,omitempty"`
+33
View File
@@ -0,0 +1,33 @@
package models
import "time"
// BrokenLinkIssue represents a broken link discovered in a post.
type BrokenLinkIssue struct {
URL string `json:"url"`
Status string `json:"status"` // 404, 500, timeout, error
StatusCode int `json:"status_code"` // HTTP status code if available
Internal bool `json:"internal"`
}
// PostBrokenLinks summarizes broken links for a single post.
type PostBrokenLinks struct {
PostID int `json:"post_id"`
PostTitle string `json:"post_title"`
PostURL string `json:"post_url"`
BrokenLinks []BrokenLinkIssue `json:"broken_links"`
TotalBroken int `json:"total_broken"`
InternalBroken int `json:"internal_broken"`
ExternalBroken int `json:"external_broken"`
}
// BrokenLinkReport is a snapshot of broken links across posts.
type BrokenLinkReport struct {
ID string `json:"id"`
CreatedAt time.Time `json:"created_at"`
DurationMs int64 `json:"duration_ms"`
TotalPosts int `json:"total_posts"`
LinksChecked int `json:"links_checked"`
TotalBrokenLinks int `json:"total_broken_links"`
Posts []PostBrokenLinks `json:"posts"`
}
+54
View File
@@ -0,0 +1,54 @@
package models
import "time"
// MetaScore represents a simple SEO meta score and issues.
type MetaScore struct {
Title string `json:"title"`
Description string `json:"description"`
TitleScore int `json:"title_score"`
DescriptionScore int `json:"description_score"`
TitleIssues []string `json:"title_issues"`
DescriptionIssues []string `json:"description_issues"`
}
// ExternalImageIssue represents an image hosted on an external domain.
type ExternalImageIssue struct {
URL string `json:"url"`
Host string `json:"host"`
Source string `json:"source"` // src or srcset
}
// PostImageAudit summarizes external image findings for a post.
type PostImageAudit struct {
PostID int `json:"post_id"`
PostTitle string `json:"post_title"`
PostURL string `json:"post_url"`
ExternalCount int `json:"external_count"`
ExternalImages []ExternalImageIssue `json:"external_images"`
Meta *MetaScore `json:"meta,omitempty"`
}
// FeaturedImageMismatch represents a mismatch between EN and ES featured images.
type FeaturedImageMismatch struct {
EnglishPostID int `json:"english_post_id"`
SpanishPostID int `json:"spanish_post_id"`
EnglishTitle string `json:"english_title"`
SpanishTitle string `json:"spanish_title"`
EnglishURL string `json:"english_url"`
SpanishURL string `json:"spanish_url"`
EnglishFeaturedID int `json:"english_featured_id"`
SpanishFeaturedID int `json:"spanish_featured_id"`
}
// ImageAuditReport is a snapshot of external image issues and meta health.
type ImageAuditReport struct {
ID string `json:"id"`
CreatedAt time.Time `json:"created_at"`
DurationMs int64 `json:"duration_ms"`
TotalPosts int `json:"total_posts"`
ImagesChecked int `json:"images_checked"`
TotalExternalImages int `json:"total_external_images"`
Posts []PostImageAudit `json:"posts"`
FeaturedImageMismatches []FeaturedImageMismatch `json:"featured_image_mismatches"`
}
+24
View File
@@ -0,0 +1,24 @@
package models
import "time"
// OrphanPost represents a post missing a translation.
type OrphanPost struct {
PostID int `json:"post_id"`
Title string `json:"title"`
URL string `json:"url"`
Language string `json:"language"`
MissingLang string `json:"missing_lang"`
ModifiedAt time.Time `json:"modified_at"`
}
// TranslationDraft represents a translated draft payload.
type TranslationDraft struct {
Language string `json:"language"`
Title string `json:"title"`
Slug string `json:"slug"`
SEOTitle string `json:"seo_title"`
MetaDescription string `json:"meta_description"`
Excerpt string `json:"excerpt"`
ContentHTML string `json:"content_html"`
}