Skip to content

Implement lock-free versioned (revisionId) and Set-based Redis caching for /tasks #5103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
eyebrowsoffire wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Implement lock-free versioned (revisionId) and Set-based Redis caching for /tasks #5103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7c1e52f
Implement lock-free versioned (revisionId) and Set-based Redis cachin…
eyebrowsoffire Jul 10, 2026
ccb37df
Merge upstream/main into tasks_cache and resolve conflicts
eyebrowsoffire Jul 10, 2026
a5a92d1
Addressed Gemini comments.
eyebrowsoffire Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion app_dart/bin/gae_server.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ Future<void> main() async {
}

final cache = CacheService.redis();
final firestore = await FirestoreService.from(const GoogleAuthProvider());
final firestore = await FirestoreService.from(
const GoogleAuthProvider(),
cache: cache,
);
final bigQuery = await BigQueryService.from(const GoogleAuthProvider());

// Start with a fresh copy of the DynamicConfig. If this throws, the server
Expand Down
62 changes: 61 additions & 1 deletion app_dart/lib/src/model/firestore/task.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
/// @docImport 'commit.dart';
library;

import 'dart:convert';
import 'dart:typed_data';

import 'package:buildbucket/buildbucket_pb.dart' as bbv2;
import 'package:cocoon_common/task_status.dart';
import 'package:googleapis/firestore/v1.dart' hide Status;
Expand Down Expand Up @@ -113,6 +116,7 @@ final class Task extends AppDocument<Task> {
static const fieldStatus = 'status';
static const fieldTestFlaky = 'testFlaky';
static const fieldAttempt = 'attempt';
static const fieldRevisionId = 'revisionId';

/// Returns a document ID for a task from the given parameters.
static AppDocumentId<Task> documentIdFor({
Expand All @@ -136,17 +140,52 @@ final class Task extends AppDocument<Task> {
fromDocument: Task.fromDocument,
);

static Uint8List _serializeTask(Task task) {
return Uint8List.fromList(
utf8.encode(
json.encode(Document(name: task.name, fields: task.fields).toJson()),
),
);
}

static Task _deserializeTask(Uint8List data) {
final jsonMap = json.decode(utf8.decode(data)) as Map<String, dynamic>;
return Task.fromDocument(Document.fromJson(jsonMap));
}

Comment on lines +143 to +155

@jtmcdole jtmcdole Jul 16, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are duplicates in firestore.dart.

/// Lookup [Task] from Firestore.
///
/// `documentName` follows `/projects/{project}/databases/{database}/documents/{document_path}`
static Future<Task> fromFirestore(
FirestoreService firestoreService,
AppDocumentId<Task> id,
) async {
if (firestoreService.cache != null) {
final cachedTask = await firestoreService.cache!.get(
'tasks',
id.documentId,
);
if (cachedTask != null) {
return _deserializeTask(cachedTask);
}
}

final document = await firestoreService.getDocument(
p.posix.join(kDatabase, 'documents', kTaskCollectionId, id.documentId),
);
return Task.fromDocument(document);
final task = Task.fromDocument(document);

if (firestoreService.cache != null) {
await firestoreService.cache!.insertVersioned('tasks', [
VersionedCacheEntry(
key: id.documentId,
value: _serializeTask(task),
revisionId: task.revisionId,
ttl: const Duration(hours: 12),
),
]);
}
return task;
}

factory Task({
Expand Down Expand Up @@ -178,6 +217,7 @@ final class Task extends AppDocument<Task> {
fieldStatus: status.value.toValue(),
fieldTestFlaky: testFlaky.toValue(),
fieldAttempt: currentAttempt.toValue(),
fieldRevisionId: 1.toValue(),
},
name: p.posix.join(
kDatabase,
Expand Down Expand Up @@ -229,12 +269,26 @@ final class Task extends AppDocument<Task> {
fields: {fieldStatus: Value(stringValue: status.value)},
),
updateMask: DocumentMask(fieldPaths: [fieldStatus]),
updateTransforms: [
FieldTransform(
fieldPath: fieldRevisionId,
increment: Value(integerValue: '1'),
),
],
);
}

/// The task was run successfully.
static const statusSucceeded = TaskStatus.succeeded;

int get revisionId => fields.containsKey(fieldRevisionId)
? int.parse(fields[fieldRevisionId]!.integerValue!)
: 1;

void incrementRevisionId() {
fields[fieldRevisionId] = (revisionId + 1).toValue();
}

/// The timestamp (in milliseconds since the Epoch) that this task was
/// created.
///
Expand Down Expand Up @@ -308,14 +362,17 @@ final class Task extends AppDocument<Task> {

void setStatus(TaskStatus status) {
fields[fieldStatus] = status.value.toValue();
incrementRevisionId();
}

void setEndTimestamp(int endTimestamp) {
fields[fieldEndTimestamp] = endTimestamp.toValue();
incrementRevisionId();
}

void setTestFlaky(bool testFlaky) {
fields[fieldTestFlaky] = testFlaky.toValue();
incrementRevisionId();
}

void updateFromBuild(bbv2.Build build) {
Expand All @@ -335,6 +392,7 @@ final class Task extends AppDocument<Task> {
.toValue();

_setStatusFromLuciStatus(build);
incrementRevisionId();
}

void resetAsRetry({int? attempt, DateTime? now}) {
Expand All @@ -360,11 +418,13 @@ final class Task extends AppDocument<Task> {
fieldTestFlaky: false.toValue(),
fieldCommitSha: commitSha.toValue(),
fieldAttempt: attempt.toValue(),
fieldRevisionId: 1.toValue(),
};
}

void setBuildNumber(int buildNumber) {
fields[fieldBuildNumber] = buildNumber.toValue();
incrementRevisionId();
}

void _setStatusFromLuciStatus(bbv2.Build build) {
Expand Down
Loading
Loading