Skip to content

Commit bae16ed

Browse files
ejsmithniemyjski
ejsmith
and
niemyjski
authored
Stack filtering improvements and bug fixes (#825)
* #746 - Add more test coverage for resolving new stack counts * Fixed some bugs with test data builder * Update deps. Some k8s updates. * Progress on new stack filter issue. * More progress * Don't apply retention filter to stack id filter when inverted * Got the main test passing. Not sure if the other failures are correct or not. * Fixed one failing unit test * Some minor changes * Disable AD Windows build warnings #493 * Added ability to generate many events using TotalOccurrences * Refactored how additional events are created. * Added new test for posting null session identity name * WIP - Event Stack Filter Tests * Update Deps * Fixed some build messages * Fix issue with message bus broker async fire and forget. * Working on stack inverting issues * Update ES docker to 7.12 * Progress in stack filter refactor * Fix a couple tests * Fixing more tests * Fix remaining tests. Update repos. * Remove repos and parser projects * Update deps / respond to feedback Co-authored-by: Blake Niemyjski <[email protected]>
1 parent a11889e commit bae16ed

40 files changed

+901
-466
lines changed

‎.github/workflows/build.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
run: "echo ref: ${{github.ref}} event: ${{github.event_name}}"
2020
- name: Build Version
2121
run: |
22-
dotnet tool install --global minver-cli --version 2.3.1
22+
dotnet tool install --global minver-cli --version 2.5.0
2323
version=$(minver --tag-prefix v)
2424
echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV
2525
echo "VERSION=$version" >> $GITHUB_ENV

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -126,3 +126,5 @@ k8s/ex-prod\.yaml
126126
*secrets*
127127
k8s/ex-*-snapshots.yaml
128128
node_modules
129+
130+
*.DotSettings

‎Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ CMD [ "dotnet", "Exceptionless.Web.dll" ]
7878

7979
# completely self-contained
8080

81-
FROM exceptionless/elasticsearch:7.10.0 AS exceptionless
81+
FROM exceptionless/elasticsearch:7.12.0 AS exceptionless
8282

8383
WORKDIR /app
8484
COPY --from=api-publish /app/src/Exceptionless.Web/out ./

‎Exceptionless.sln

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
1313
CONTRIBUTING.md = CONTRIBUTING.md
1414
docker-compose.yml = docker-compose.yml
1515
Dockerfile = Dockerfile
16+
build\docker\elasticsearch\Dockerfile = build\docker\elasticsearch\Dockerfile
1617
global.json = global.json
1718
LICENSE.txt = LICENSE.txt
1819
NuGet.Config = NuGet.Config
@@ -58,7 +59,6 @@ Global
5859
HideSolutionNode = FALSE
5960
EndGlobalSection
6061
GlobalSection(ExtensibilityGlobals) = postSolution
61-
GlobalSection(ExtensibilityGlobals) = postSolution
6262
SolutionGuid = {1A90AFA5-B81C-4B1B-9DFA-2D90F8CA0EF0}
6363
EndGlobalSection
6464
EndGlobal

‎build/common.props

+3-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<Product>Exceptionless</Product>
66
<MinVerSkip Condition="'$(Configuration)' == 'Debug'">true</MinVerSkip>
77
<MinVerTagPrefix>v</MinVerTagPrefix>
8-
<Copyright>Copyright (c) 2020 Exceptionless. All rights reserved.</Copyright>
8+
<Copyright>Copyright (c) 2021 Exceptionless. All rights reserved.</Copyright>
99
<RepositoryUrl>https://github.com/exceptionless/exceptionless</RepositoryUrl>
1010
<Authors>Exceptionless</Authors>
1111
<NoWarn>$(NoWarn);CS1591</NoWarn>
@@ -19,8 +19,8 @@
1919

2020
<ItemGroup>
2121
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All"/>
22-
<PackageReference Include="AsyncFixer" Version="1.3.0" PrivateAssets="All" />
23-
<PackageReference Include="MinVer" Version="2.3.1" PrivateAssets="All" />
22+
<PackageReference Include="AsyncFixer" Version="1.5.1" PrivateAssets="All" />
23+
<PackageReference Include="MinVer" Version="2.5.0" PrivateAssets="All" />
2424
</ItemGroup>
2525

2626
</Project>

‎build/docker/elasticsearch/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# https://www.docker.elastic.co/
2-
FROM docker.elastic.co/elasticsearch/elasticsearch:7.11.0
2+
FROM docker.elastic.co/elasticsearch/elasticsearch:7.12.0
33

44
RUN elasticsearch-plugin install -b mapper-size
55
RUN elasticsearch-plugin install -b repository-azure

‎docker-compose.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ version: '3.5'
55

66
services:
77
elasticsearch:
8-
image: exceptionless/elasticsearch:7.10.0
8+
image: exceptionless/elasticsearch:7.12.0
99
environment:
1010
discovery.type: single-node
1111
xpack.security.enabled: 'false'
@@ -17,7 +17,7 @@ services:
1717
kibana:
1818
depends_on:
1919
- elasticsearch
20-
image: docker.elastic.co/kibana/kibana:7.10.0
20+
image: docker.elastic.co/kibana/kibana:7.12.0
2121
ports:
2222
- 5601:5601
2323

‎global.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"sdk": {
33
"version": "5.0.100",
4-
"rollForward": "latestMajor"
4+
"rollForward": "latestMinor"
55
}
66
}

‎k8s/certificates.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
apiVersion: cert-manager.io/v1alpha2
1+
apiVersion: cert-manager.io/v1
22
kind: Certificate
33
metadata:
44
name: tls-secret

‎k8s/cluster-issuer.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
apiVersion: cert-manager.io/v1alpha2
1+
apiVersion: cert-manager.io/v1
22
kind: ClusterIssuer
33
metadata:
44
name: letsencrypt-prod
@@ -14,7 +14,7 @@ spec:
1414
class: nginx
1515

1616
---
17-
apiVersion: cert-manager.io/v1alpha2
17+
apiVersion: cert-manager.io/v1
1818
kind: ClusterIssuer
1919
metadata:
2020
name: selfsigned

‎k8s/ex-prod-tasks.ps1

+4-4
Original file line numberDiff line numberDiff line change
@@ -44,17 +44,16 @@ kubectl run --namespace ex-prod ex-prod-client --rm --tty -i --restart='Never' `
4444
# upgrade nginx ingress to latest
4545
# https://github.com/kubernetes/ingress-nginx/releases
4646
helm repo update
47-
helm upgrade --reset-values --namespace nginx-ingress -f nginx-values.yaml nginx-ingress stable/nginx-ingress --dry-run
47+
helm upgrade --reset-values --namespace ingress-nginx -f nginx-values.yaml ingress-nginx ingress-nginx/ingress-nginx --dry-run
4848

4949
# upgrade cert-manager
5050
# https://github.com/jetstack/cert-manager/releases
5151
helm repo update
52-
kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.crds.yaml
5352
helm upgrade cert-manager jetstack/cert-manager --namespace cert-manager --reset-values --set ingressShim.defaultIssuerName=letsencrypt-prod --set ingressShim.defaultIssuerKind=ClusterIssuer --dry-run
5453

5554
# upgrade dashboard
5655
# https://github.com/kubernetes/dashboard/releases
57-
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml
56+
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.1.0/aio/deploy/recommended.yaml
5857

5958
# upgrade kubecost
6059
helm repo update
@@ -63,11 +62,12 @@ helm upgrade kubecost kubecost/cost-analyzer --namespace kubecost --reset-values
6362
# upgrade goldilocks
6463
helm repo update
6564
helm upgrade goldilocks fairwinds-stable/goldilocks --namespace goldilocks --reset-values --dry-run
65+
helm upgrade vpa fairwinds-stable/vpa --namespace vpa -f vpa-values.yaml --reset-values --dry-run
6666

6767
# upgrade elasticsearch operator
6868
# https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html
6969
# https://github.com/elastic/cloud-on-k8s/releases
70-
kubectl apply -f https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml
70+
kubectl apply -f https://download.elastic.co/downloads/eck/1.3.1/all-in-one.yaml
7171

7272
# upgrade elasticsearch
7373
kubectl apply --namespace ex-prod -f ex-prod-elasticsearch.yaml

‎k8s/ex-setup.ps1

+9-6
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ choco install azure-cli
1010

1111
# install helm
1212
choco install kubernetes-helm
13-
helm repo add stable https://kubernetes-charts.storage.googleapis.com/
13+
helm repo add "stable" "https://charts.helm.sh/stable" --force-update
1414
helm repo add jetstack https://charts.jetstack.io
1515
helm repo update
1616

@@ -48,7 +48,7 @@ az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER --overwr
4848

4949
# install dashboard
5050
# https://github.com/kubernetes/dashboard/releases
51-
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.3/aio/deploy/recommended.yaml
51+
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.1.0/aio/deploy/recommended.yaml
5252

5353
# create admin user to login to the dashboard
5454
kubectl apply -f admin-service-account.yaml
@@ -59,7 +59,7 @@ kubectl config set-context --current --namespace=ex-$ENV
5959
# setup elasticsearch operator
6060
# https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html
6161
# https://github.com/elastic/cloud-on-k8s/releases
62-
kubectl apply -f https://download.elastic.co/downloads/eck/1.2.1/all-in-one.yaml
62+
kubectl apply -f https://download.elastic.co/downloads/eck/1.3.1/all-in-one.yaml
6363

6464
# view ES operator logs
6565
kubectl -n elastic-system logs -f statefulset.apps/elastic-operator
@@ -98,11 +98,11 @@ curl -X PUT -H "Content-Type: application/json" -k `
9898
Remove-Job $ELASTIC_JOB
9999

100100
# install nginx ingress
101-
helm install nginx-ingress stable/nginx-ingress --namespace nginx-ingress --values nginx-values.yaml
101+
helm install --namespace ingress-nginx -f nginx-values.yaml ingress-nginx ingress-nginx/ingress-nginx
102102

103103
# wait for external ip to be assigned
104-
kubectl get service -l app=nginx-ingress --namespace nginx-ingress
105-
$IP=$(kubectl get service -l app=nginx-ingress --namespace nginx-ingress -o=jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
104+
kubectl get service -l app.kubernetes.io/name=ingress-nginx --namespace ingress-nginx
105+
$IP=$(kubectl get service -l app.kubernetes.io/name=ingress-nginx --namespace ingress-nginx -o=jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
106106
$PUBLICIPID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$IP')].[id]" --output tsv)
107107
az network public-ip update --ids $PUBLICIPID --dns-name $CLUSTER
108108

@@ -117,11 +117,14 @@ helm install cert-manager jetstack/cert-manager --namespace cert-manager --set i
117117
# https://kubecost.com/install?ref=home
118118
kubectl create namespace kubecost
119119
helm repo add kubecost https://kubecost.github.io/cost-analyzer/
120+
$KUBECOST_KEY=""
120121
helm install kubecost kubecost/cost-analyzer --namespace kubecost --set kubecostToken=$KUBECOST_KEY
121122

122123
# install goldilocks
123124
helm repo add fairwinds-stable https://charts.fairwinds.com/stable
125+
helm install vpa fairwinds-stable/vpa --namespace vpa --create-namespace -f vpa-values.yaml
124126
helm install goldilocks fairwinds-stable/goldilocks --namespace goldilocks
127+
kubectl label ns ex-$ENV goldilocks.fairwinds.com/enabled=true
125128

126129
# TODO: update this file using the cluster name for the dns
127130
kubectl apply -f certificates.yaml

‎k8s/vpa-values.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
recommender:
2+
extraArgs:
3+
prometheus-address: |
4+
http://kubecost-prometheus-server.kubecost.svc.cluster.local
5+
storage: prometheus

‎src/Exceptionless.Core/Authentication/ActiveDirectoryLoginProvider.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System.DirectoryServices;
1+
# pragma warning disable CA1416
2+
using System.DirectoryServices;
23
using Exceptionless.Core.Configuration;
34

45
namespace Exceptionless.Core.Authentication {
@@ -60,4 +61,5 @@ private SearchResult FindUser(string username) {
6061
}
6162
}
6263
}
63-
}
64+
}
65+
# pragma warning restore CA1416

‎src/Exceptionless.Core/Exceptionless.Core.csproj

+10-10
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,20 @@
2323
<ItemGroup>
2424
<PackageReference Include="AutoMapper" Version="10.1.1" />
2525
<PackageReference Include="AutoMapper.Collection" Version="7.0.1" />
26-
<PackageReference Include="FluentValidation" Version="9.3.0" />
27-
<PackageReference Include="Foundatio.Extensions.Hosting" Version="10.0.1" />
28-
<PackageReference Include="Foundatio.JsonNet" Version="10.0.1" />
29-
<PackageReference Include="NEST.JsonNetSerializer" Version="7.10.0" />
30-
<PackageReference Include="Handlebars.Net" Version="1.11.5" />
31-
<PackageReference Include="McSherry.SemanticVersioning" Version="1.4.0" />
26+
<PackageReference Include="FluentValidation" Version="9.5.3" />
27+
<PackageReference Include="Foundatio.Extensions.Hosting" Version="10.1.1" />
28+
<PackageReference Include="Foundatio.JsonNet" Version="10.1.1" />
29+
<PackageReference Include="NEST.JsonNetSerializer" Version="7.12.0" />
30+
<PackageReference Include="Handlebars.Net" Version="2.0.7" />
31+
<PackageReference Include="McSherry.SemanticVersioning" Version="1.4.1" />
3232
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
33-
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="5.0.0" />
33+
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="5.0.5" />
3434
<PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
35-
<PackageReference Include="Stripe.net" Version="39.27.0" />
35+
<PackageReference Include="Stripe.net" Version="39.44.0" />
3636
<PackageReference Include="System.DirectoryServices" Version="5.0.0" />
37-
<PackageReference Include="UAParser" Version="3.1.44" />
37+
<PackageReference Include="UAParser" Version="3.1.46" />
3838

39-
<PackageReference Include="Foundatio.Repositories.Elasticsearch" Version="7.10.0" Condition="'$(ReferenceFoundatioRepositoriesSource)' == '' OR '$(ReferenceFoundatioRepositoriesSource)' == 'false'" />
39+
<PackageReference Include="Foundatio.Repositories.Elasticsearch" Version="7.12.1" Condition="'$(ReferenceFoundatioRepositoriesSource)' == '' OR '$(ReferenceFoundatioRepositoriesSource)' == 'false'" />
4040
<ProjectReference Include="..\..\..\Foundatio.Repositories\src\Foundatio.Repositories.Elasticsearch\Foundatio.Repositories.Elasticsearch.csproj" Condition="'$(ReferenceFoundatioRepositoriesSource)' == 'true'" />
4141
</ItemGroup>
4242
</Project>

‎src/Exceptionless.Core/Extensions/QueryNodeExtensions.cs

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using System;
2+
using Foundatio.Parsers.LuceneQueries.Nodes;
3+
4+
namespace Exceptionless.Core.Extensions {
5+
public static class QueryNodeExtensions {
6+
public static GroupNode GetParent(this IQueryNode node, Func<GroupNode, bool> condition) {
7+
if (node == null)
8+
return null;
9+
10+
IQueryNode queryNode = node;
11+
do {
12+
GroupNode groupNode = queryNode as GroupNode;
13+
if (groupNode != null && condition(groupNode))
14+
return groupNode;
15+
16+
queryNode = queryNode.Parent;
17+
}
18+
while (queryNode != null);
19+
20+
return null;
21+
}
22+
}
23+
}

‎src/Exceptionless.Core/Mail/Mailer.cs

+8-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
namespace Exceptionless.Core.Mail {
1919
public class Mailer : IMailer {
20-
private readonly ConcurrentDictionary<string, Func<object, string>> _cachedTemplates = new ConcurrentDictionary<string, Func<object, string>>();
20+
private readonly ConcurrentDictionary<string, HandlebarsTemplate<object, object>> _cachedTemplates = new ConcurrentDictionary<string, HandlebarsTemplate<object, object>>();
2121
private readonly IQueue<MailMessage> _queue;
2222
private readonly FormattingPluginManager _pluginManager;
2323
private readonly AppOptions _appOptions;
@@ -134,10 +134,11 @@ public Task SendOrganizationInviteAsync(User sender, Organization organization,
134134
{ "InviteToken", invite.Token }
135135
};
136136

137+
var body = RenderTemplate(template, data);
137138
return QueueMessageAsync(new MailMessage {
138139
To = invite.EmailAddress,
139140
Subject = subject,
140-
Body = RenderTemplate(template, data)
141+
Body = body
141142
}, template);
142143
}
143144

@@ -261,18 +262,20 @@ public Task SendUserPasswordResetAsync(User user) {
261262

262263
private string RenderTemplate(string name, IDictionary<string, object> data) {
263264
var template = GetCompiledTemplate(name);
264-
return template(data);
265+
var result = template(data);
266+
return result?.ToString();
265267
}
266268

267-
private Func<object, string> GetCompiledTemplate(string name) {
269+
private HandlebarsTemplate<object, object> GetCompiledTemplate(string name) {
268270
return _cachedTemplates.GetOrAdd(name, templateName => {
269271
var assembly = typeof(Mailer).Assembly;
270272
string resourceName = $"Exceptionless.Core.Mail.Templates.{templateName}.html";
271273

272274
using (var stream = assembly.GetManifestResourceStream(resourceName)) {
273275
using (var reader = new StreamReader(stream)) {
274276
string template = reader.ReadToEnd();
275-
return Handlebars.Compile(template);
277+
var compiledTemplateFunc = Handlebars.Compile(template);
278+
return compiledTemplateFunc;
276279
}
277280
}
278281
});

‎src/Exceptionless.Core/Models/Stack.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
using Newtonsoft.Json.Converters;
99

1010
namespace Exceptionless.Core.Models {
11-
[DebuggerDisplay("Id: {Id}, Type: {Type}, Title: {Title}, TotalOccurrences: {TotalOccurrences}")]
11+
[DebuggerDisplay("Id={Id} Type={Type} Status={Status} IsDeleted={IsDeleted} Title={Title} TotalOccurrences={TotalOccurrences}")]
1212
public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes {
1313
public Stack() {
1414
Tags = new TagSet();

‎src/Exceptionless.Core/Models/StackSummaryModel.cs

+2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System;
2+
using System.Diagnostics;
23

34
namespace Exceptionless.Core.Models {
5+
[DebuggerDisplay("Id: {Id}, Status: {Status}, Title: {Title}, First: {FirstOccurrence}, Last: {LastOccurrence}")]
46
public class StackSummaryModel : SummaryData {
57
public string Id { get; set; }
68
public string Title { get; set; }

‎src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs

-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
using System.Collections.Generic;
33
using System.Threading.Tasks;
44
using Exceptionless.Core.Configuration;
5-
using Exceptionless.Core.Extensions;
65
using Exceptionless.Core.Models;
76
using Exceptionless.Core.Models.Data;
87
using Exceptionless.Core.Repositories.Queries;

‎src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs

+5-5
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,11 @@ public AppFilterQueryBuilder(AppOptions options) {
103103
}
104104

105105
var index = ctx.Options.GetElasticIndex();
106-
bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index);
106+
bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index, ctx);
107107
string field = shouldApplyRetentionFilter ? GetDateField(index) : null;
108108

109109
if (sfq.Stack != null) {
110-
var stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName;
110+
string stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName;
111111
var organization = allowedOrganizations.SingleOrDefault(o => o.Id == sfq.Stack.OrganizationId);
112112
if (organization != null) {
113113
if (shouldApplyRetentionFilter)
@@ -161,13 +161,13 @@ public AppFilterQueryBuilder(AppOptions options) {
161161
return Query<T>.DateRange(r => r.Field(field).GreaterThanOrEquals($"now/d-{(int)retentionDays}d").LessThanOrEquals("now/d+1d"));
162162
}
163163

164-
private bool ShouldApplyRetentionFilter(IIndex index) {
164+
private bool ShouldApplyRetentionFilter<T>(IIndex index, QueryBuilderContext<T> ctx) where T : class, new() {
165165
if (index == null)
166166
throw new ArgumentNullException(nameof(index));
167-
167+
168168
var indexType = index.GetType();
169169
if (indexType == typeof(StackIndex))
170-
return true;
170+
return !ctx.Source.IsEventStackFilterInverted();
171171

172172
if (indexType == typeof(EventIndex))
173173
return true;

0 commit comments

Comments
 (0)