Improve Active Workers table and recover stale tasks

Backend:
- Add _recover_stale_tasks() to reset tasks stuck in 'in_progress'
  from previous crashes (tasks >5 min old get reset to pending)
- Called automatically on startup

Frontend:
- Fix dark mode colors using CSS variables instead of hardcoded values
- Add elapsed time column showing how long task has been running
- Add spinning indicator next to package name
- Add status badge (Running/Stale?)
- Highlight stale tasks (>5 min) in amber
- Auto-updates every 5 seconds with existing refresh
This commit is contained in:
Mondo Diaz
2026-02-02 14:29:17 -06:00
parent 1138309aaa
commit 47b137f4eb
3 changed files with 143 additions and 23 deletions

View File

@@ -33,6 +33,45 @@ _cache_worker_running: bool = False
_dispatcher_thread: Optional[threading.Thread] = None _dispatcher_thread: Optional[threading.Thread] = None
def _recover_stale_tasks():
"""
Recover tasks stuck in 'in_progress' state from a previous crash.
Called on startup to reset tasks that were being processed when
the server crashed. Resets them to 'pending' so they can be retried.
"""
db = SessionLocal()
try:
# Find tasks that have been in_progress for more than 5 minutes
# These are likely from a crashed worker
stale_threshold = datetime.utcnow() - timedelta(minutes=5)
stale_count = (
db.query(PyPICacheTask)
.filter(
PyPICacheTask.status == "in_progress",
or_(
PyPICacheTask.started_at == None,
PyPICacheTask.started_at < stale_threshold,
),
)
.update(
{
"status": "pending",
"started_at": None,
}
)
)
db.commit()
if stale_count > 0:
logger.warning(f"Recovered {stale_count} stale in_progress tasks from previous crash")
except Exception as e:
logger.error(f"Error recovering stale tasks: {e}")
finally:
db.close()
def init_cache_worker_pool(max_workers: Optional[int] = None): def init_cache_worker_pool(max_workers: Optional[int] = None):
""" """
Initialize the cache worker pool. Called on app startup. Initialize the cache worker pool. Called on app startup.
@@ -47,6 +86,9 @@ def init_cache_worker_pool(max_workers: Optional[int] = None):
logger.warning("Cache worker pool already initialized") logger.warning("Cache worker pool already initialized")
return return
# Recover any stale tasks from previous crash before starting workers
_recover_stale_tasks()
workers = max_workers or settings.PYPI_CACHE_WORKERS workers = max_workers or settings.PYPI_CACHE_WORKERS
_cache_worker_pool = ThreadPoolExecutor( _cache_worker_pool = ThreadPoolExecutor(
max_workers=workers, max_workers=workers,

View File

@@ -280,14 +280,14 @@
display: flex; display: flex;
align-items: center; align-items: center;
gap: 0.5rem; gap: 0.5rem;
color: #3b82f6; color: var(--color-primary, #3b82f6);
margin-top: 1rem; margin-top: 1rem;
} }
.pulse-indicator { .pulse-indicator {
width: 8px; width: 8px;
height: 8px; height: 8px;
background-color: #3b82f6; background-color: var(--color-primary, #3b82f6);
border-radius: 50%; border-radius: 50%;
animation: pulse 1.5s ease-in-out infinite; animation: pulse 1.5s ease-in-out infinite;
} }
@@ -304,21 +304,21 @@
} }
.jobs-table.active-table { .jobs-table.active-table {
border: 1px solid #3b82f6; border: 1px solid var(--color-primary, #3b82f6);
border-radius: 4px; border-radius: 4px;
} }
.jobs-table.active-table th { .jobs-table.active-table th {
background: #eff6ff; background: var(--bg-tertiary);
color: #1d4ed8; color: var(--color-primary, #3b82f6);
} }
.jobs-table .active-row { .jobs-table .active-row {
background-color: #f0f9ff; background-color: var(--bg-secondary);
} }
.jobs-table .active-row:hover { .jobs-table .active-row:hover {
background-color: #e0f2fe; background-color: var(--bg-tertiary);
} }
.jobs-table .version-constraint { .jobs-table .version-constraint {
@@ -327,6 +327,66 @@
color: var(--text-secondary); color: var(--text-secondary);
} }
/* Spinner for active tasks */
.spinner {
display: inline-block;
width: 12px;
height: 12px;
border: 2px solid var(--border-color);
border-top-color: var(--color-primary, #3b82f6);
border-radius: 50%;
animation: spin 1s linear infinite;
margin-right: 0.5rem;
vertical-align: middle;
}
@keyframes spin {
to {
transform: rotate(360deg);
}
}
/* Elapsed time */
.elapsed-time {
font-family: monospace;
font-size: 0.9rem;
color: var(--text-secondary);
}
.elapsed-time.stale {
color: #f59e0b;
font-weight: 500;
}
/* Status badges */
.status-badge {
display: inline-block;
padding: 0.2rem 0.5rem;
border-radius: 4px;
font-size: 0.75rem;
font-weight: 500;
text-transform: uppercase;
}
.status-badge.running {
background-color: rgba(59, 130, 246, 0.15);
color: var(--color-primary, #3b82f6);
}
.status-badge.stale {
background-color: rgba(245, 158, 11, 0.15);
color: #f59e0b;
}
/* Stale row highlighting */
.jobs-table .stale-row {
background-color: rgba(245, 158, 11, 0.1);
}
.jobs-table .stale-row:hover {
background-color: rgba(245, 158, 11, 0.15);
}
/* Responsive */ /* Responsive */
@media (max-width: 768px) { @media (max-width: 768px) {
.status-cards { .status-cards {

View File

@@ -212,25 +212,43 @@ function AdminJobsPage() {
<th>Version</th> <th>Version</th>
<th>Depth</th> <th>Depth</th>
<th>Attempt</th> <th>Attempt</th>
<th>Started</th> <th>Elapsed</th>
<th>Status</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{activeTasks.map((task) => ( {activeTasks.map((task) => {
<tr key={task.id} className="active-row"> const elapsed = task.started_at
<td className="package-name">{task.package}</td> ? Math.floor((Date.now() - new Date(task.started_at).getTime()) / 1000)
: 0;
const minutes = Math.floor(elapsed / 60);
const seconds = elapsed % 60;
const elapsedStr = minutes > 0
? `${minutes}m ${seconds}s`
: `${seconds}s`;
const isStale = elapsed > 300; // 5 minutes
return (
<tr key={task.id} className={`active-row ${isStale ? 'stale-row' : ''}`}>
<td className="package-name">
<span className="spinner"></span>
{task.package}
</td>
<td className="version-constraint"> <td className="version-constraint">
{task.version_constraint || '*'} {task.version_constraint || '*'}
</td> </td>
<td>{task.depth}</td> <td>{task.depth}</td>
<td>{task.attempts + 1}</td> <td>{task.attempts + 1}</td>
<td className="timestamp"> <td className={`elapsed-time ${isStale ? 'stale' : ''}`}>
{task.started_at {elapsedStr}
? new Date(task.started_at).toLocaleTimeString() </td>
: '-'} <td className="status-cell">
<span className={`status-badge ${isStale ? 'stale' : 'running'}`}>
{isStale ? 'Stale?' : 'Running'}
</span>
</td> </td>
</tr> </tr>
))} );
})}
</tbody> </tbody>
</table> </table>
</div> </div>