Skip to content

Product ID key issue; sparse columns; 0 distances; row group size #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kelly-nelson wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

Product ID key issue; sparse columns; 0 distances; row group size #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0d29831
Fixing 0 widths, bad product key and sparse columns
kelly-nelson Aug 4, 2026
446125f
Merge remote-tracking branch 'origin/develop' into develop
kelly-nelson Aug 4, 2026
968d12d
Cosmetic change on row group size; snapping to gdal default.
kelly-nelson Aug 11, 2026
dde2d80
Further cleanup of distance and sparse values.
kelly-nelson Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion StandardPlugin/ADAPTParquet.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace AgGateway.ADAPT.StandardPlugin
{
internal class ADAPTParquetWriter
{
const int RowGroupSize = 65535;
const int RowGroupSize = 65536;

public ADAPTParquetWriter(ADAPTParquetColumnData columnData)
{
Expand Down
9 changes: 5 additions & 4 deletions StandardPlugin/Extensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ public static double HeadingRight(this double heading)
public static Polygon AsCoveragePolygon(this Point leadingPoint, double width, ref LeadingEdge latestLeadingEdge, double heading, double? reportedDistance, double? calculatedDistance)
{
LeadingEdge priorLeadingEdge = latestLeadingEdge;
latestLeadingEdge = new LeadingEdge(leadingPoint, width, priorLeadingEdge, heading, reportedDistance);
latestLeadingEdge = new LeadingEdge(leadingPoint, width, priorLeadingEdge, heading);
Point backRight;
Point backLeft;
if (priorLeadingEdge != null)
Expand All @@ -251,15 +251,16 @@ public static Polygon AsCoveragePolygon(this Point leadingPoint, double width, r
{
//We only consider distance when we don't have a prior point to map from
double distance = 1; //1m as default without any other information (at start of data)
if (reportedDistance != null)
if (reportedDistance > 0d)
{
distance = reportedDistance.Value;
}
else if (calculatedDistance != null)
else if (calculatedDistance > 0d)
{
distance = calculatedDistance.Value;
}
distance = distance > 4 ? 4 : distance; //keep distances sane
//Clamp both ends: the floor keeps the back edge off the leading edge, which would make the polygon invalid
distance = Math.Clamp(distance, 0.1d, 4d);

backRight = latestLeadingEdge.Right.Destination(distance, HeadingBack(heading));
backLeft = backRight.Destination(width, HeadingLeft(heading));
Expand Down
16 changes: 9 additions & 7 deletions StandardPlugin/ImplementSection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,12 @@ public bool TryGetCoveragePolygon(SpatialRecord record, SpatialRecord previousRe
priorPoint = new Point(priorADAPTPoint.X, priorADAPTPoint.Y);
}

var sectionDefinitions = FactoredDefinitionsBySourceCodeByProduct[string.Empty];

double bearing = 0d;
if (FactoredDefinitionsBySourceCodeByProduct[string.Empty].ContainsKey("vrHeading"))
if (sectionDefinitions.TryGetValue("vrHeading", out var headingDefinition))
{
var headingValue = ((NumericRepresentationValue)record.GetMeterValue(FactoredDefinitionsBySourceCodeByProduct[string.Empty]["vrHeading"].WorkingData))?.Value?.Value;
var headingValue = ((NumericRepresentationValue)record.GetMeterValue(headingDefinition.WorkingData))?.Value?.Value;
if (headingValue != null)
{
bearing = headingValue.Value;
Expand All @@ -194,13 +196,13 @@ public bool TryGetCoveragePolygon(SpatialRecord record, SpatialRecord previousRe
var xy = x.Destination(Offset.Y ?? 0d, bearing + 90d % 360d);

double? reportedDistance = null;
if (FactoredDefinitionsBySourceCodeByProduct[string.Empty].ContainsKey("vrDistanceTraveled") &&
record.GetMeterValue(FactoredDefinitionsBySourceCodeByProduct[string.Empty]["vrDistanceTraveled"].WorkingData) is NumericRepresentationValue distanceData)
if (sectionDefinitions.TryGetValue("vrDistanceTraveled", out var distanceDefinition) &&
record.GetMeterValue(distanceDefinition.WorkingData) is NumericRepresentationValue distanceData)
{
reportedDistance = distanceData?.Value?.Value;
}
double? calculatedDistance = null;
if (reportedDistance == null && priorPoint != null)
double? calculatedDistance = null;
if (!(reportedDistance > 0d) && priorPoint != null)
{
calculatedDistance = GeometryExporter.HaversineDistance(priorPoint, point);
}
Expand All @@ -220,7 +222,7 @@ public void ClearLeadingEdge()

internal class LeadingEdge
{
public LeadingEdge(Point leadingPoint, double width, LeadingEdge priorLeadingEdge, double heading, double? reportedDistance)
public LeadingEdge(Point leadingPoint, double width, LeadingEdge priorLeadingEdge, double heading)
{
Heading = heading;
double wh = width / 2d;
Expand Down
23 changes: 15 additions & 8 deletions StandardPlugin/WorkRecordExporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,9 +521,9 @@ private void ExportOperationSpatialRecords(ADAPTParquetColumnData runningOutput,
{
if (dataColumn.ProductId != null && section.ProductIndexWorkingData != null)
{
if (section.FactoredDefinitionsBySourceCodeByProduct.ContainsKey(dataColumn.ProductId))
if (section.FactoredDefinitionsBySourceCodeByProduct.TryGetValue(dataColumn.ProductId, out var factoredDefinitionsForProduct) &&
factoredDefinitionsForProduct.TryGetValue(dataColumn.SrcName, out var factoredDefinition))
{
var factoredDefinition = section.FactoredDefinitionsBySourceCodeByProduct[dataColumn.ProductId][dataColumn.SrcName];
NumericRepresentationValue value = record.GetMeterValue(factoredDefinition.WorkingData) as NumericRepresentationValue;
var doubleVal = value.AsConvertedDouble(dataColumn.TargetUOMCode) * factoredDefinition.Factor;

Expand All @@ -540,15 +540,22 @@ private void ExportOperationSpatialRecords(ADAPTParquetColumnData runningOutput,
}
else
{
dataColumn.Values.Add(0d); //We've grouped operations together and this doesn't apply.
dataColumn.Values.Add(0d); //This section doesn't report this working data for this product (e.g. grouped operations, or not every section reports every column)
}
}
else
{
var factoredDefinition = section.FactoredDefinitionsBySourceCodeByProduct[string.Empty][dataColumn.SrcName];
NumericRepresentationValue value = record.GetMeterValue(factoredDefinition.WorkingData) as NumericRepresentationValue;
var doubleVal = value.AsConvertedDouble(dataColumn.TargetUOMCode) * factoredDefinition.Factor;
dataColumn.Values.Add(doubleVal);
if (section.FactoredDefinitionsBySourceCodeByProduct.TryGetValue(string.Empty, out var factoredDefinitionsForSection) &&
factoredDefinitionsForSection.TryGetValue(dataColumn.SrcName, out var factoredDefinition))
{
NumericRepresentationValue value = record.GetMeterValue(factoredDefinition.WorkingData) as NumericRepresentationValue;
var doubleVal = value.AsConvertedDouble(dataColumn.TargetUOMCode) * factoredDefinition.Factor;
dataColumn.Values.Add(doubleVal);
}
else
{
dataColumn.Values.Add(null); //This section doesn't report this working data; missing is not zero
}
}
}
}
Expand Down Expand Up @@ -600,7 +607,7 @@ internal string Key()
}
internal string ProductKey()
{
return string.Join(";", ProductIds.OrderBy(x => x).ToString());
return string.Join(";", ProductIds.OrderBy(x => x));
}

internal bool IsMatchingOperation(OperationDefinition other)
Expand Down
Loading