I would like my kusto query to remember and return, i.e. fill-down, the last non-null or non-empty value when I parse or extract a field from a log as below.
datatable (Date:datetime, LogEntry:string) [
datetime(1910-06-11), "version: 1.0",
datetime(1930-01-01), "starting foo",
datetime(1953-01-01), "ending foo",
datetime(1910-06-11), "version: 2.0",
datetime(1930-01-01), "starting foo",
datetime(1953-01-01), "ending foo" ]
| parse LogEntry with 'version: ' Version
| project Date, Version, LogEntry
Is there a way to do this?
Sorry for the delay, are you looking for something like this:
let sampleData = datatable (Date:datetime, LogEntry:string) [
datetime(1910-06-11), "version: 1.1",
datetime(1930-01-01), "starting foo",
datetime(1953-01-01), "ending foo",
datetime(1910-06-11), "version: 2.1",
datetime(1930-01-01), "starting foo",
datetime(1953-01-01), "ending foo",
datetime(1950-01-01), "version: 3.1"]
| parse LogEntry with 'version: ' Version:double
| order by Date asc
| extend rn = row_number()
| extend rn = iif(isempty(Version),0 , rn)
| extend rn = row_cumsum(rn, rn!=0);
sampleData
| project-away Version
| lookup (sampleData | where isnotempty(Version) ) on rn
| project-away Date1, LogEntry1, rn
here is the output:
Related
I would like to ask for help. I need to split values from key "Text" on base space " " and join to one line. In actually code I calculate with exactly position but if key Text has S10 is show only S1.
My input
[
{
"PartNumber": "5SE32DFVLG002",
"ClassificationNo": "500001",
"StringValue": "R0050SWSW",
"Field": "95001",
"Text": "S1 W1 cr.sec+colour"
},
{
"PartNumber": "5SE32DFVLG002",
"ClassificationNo": "500001",
"StringValue": "R0050SWSW",
"Field": "95004",
"Text": "S1 W10 cr.sec+colour"
}
]
My actually condition in jq play
[.Oslm[] | select(.ClassificationNo=="500001" and .StringValue!="") |
{PartNumber,ClassificationNo,StringValue,Field,Text}] |
sort_by(.Field) | .[] | [.PartNumber,.ClassificationNo,
.Field[3:5],.Text[0:2] + "-" + .Text[3:5] + .StringValue[0:1], "Test
", .StringValue[1:10]] | join(";")
Actual result
5SE32DFVLG002;500001;95001;S1-W1R;TEST;0050SWSW
5SE32DFVLG002;500001;95004;S1-W1R;TEST;0050SWSW
I would like to have this result
5SE32DFVLG002;500001;95001;S1-W1R;TEST;0050SWSW
5SE32DFVLG002;500001;95004;S1-W10R;TEST;0050SWSW
Modify the part involving generation of .Text to something simpler using split() method in jq that can be used to split on a single white-space. This way, you are not reliant on the length of the sub-fields you want to extract
( .Text | split(" ") | .[0] + "-" + .[1] ) + .StringValue[0:1]
i.e. with full code
.[] | [ select( .ClassificationNo =="500001" and .StringValue != "" ) |
{
PartNumber,
ClassificationNo,
StringValue,
Field,
Text
} ] |
sort_by(.Field) |
map(
.PartNumber,
.ClassificationNo,
.Field[3:5],
( .Text | split(" ") | .[0] + "-" + .[1] ) + .StringValue[0:1],
"Test", .StringValue[1:10]
) |
join(";")
demo at jqplay
helo iam new to kusto query can i get the query for yesterday's date in log analytics workbook ?
let PGBARI1_Extraction_st = ADFPipelineRun
| where parse_json(Parameters).Process_Unique_RunID startswith "PGBARI1"
| where End >= Start
| where Start >= todatetime (strcat (tostring(format_datetime(now(),'yyyy-MM-dd'))))
| summarize time1 = min(Start) by TenantId , process_name = 'PGBARI1';
let PGBARI1_Compaction_end = ADFPipelineRun
| where Parameters contains "PGBARI1"
| where _ResourceId endswith "df-pr-dna-eus2"
| where OperationName contains "orchestrator"
| where End >= Start
| where Start >= todatetime (strcat (tostring(format_datetime(now(),'yyyy-MM-dd'))))
| summarize time2 = max(End) by TenantId , process_name = 'PGBARI1';
let PGBARI1 = PGBARI1_Extraction_st | join PGBARI1_Compaction_end on $left.process_name==$right.process_name
| summarize Runtime_in_minutes = sum( toint(datetime_diff('minute',time2,time1))), process_name = 'PGBARI1';
let PGBARI1_1 = ADFPipelineRun
| where Parameters contains "PGBARI1"
| where End >= Start
| where Start >= todatetime (strcat (tostring(format_datetime(now(),'yyyy-MM-dd'))))
| where Status != 'Succeeded'
| summarize Failed = count(Status);
let PGBARI_Status = PGBARI1_1 | project Process_name = 'PGBARI1', Status = iff(Failed>0,'Failed','Succeeded');
PGBARI1 | union PGBARI_Status
I need the messages in Azure AppInsights grouped by the existence of particular substrings in the messages and the counts of these messages.
At the end, here is what the grouping would look like
messages count
-------- -------
foomessages <say, 300>
barmessages <say, 450>
:
:
where
foomessages = All messages containing the substring "foo" etc.
How can I construct a query for this ?
datatable(log: string) [
"hello world",
"this is a test",
"this is a world test",
"another test"
]
| summarize
LogsWithWorld = countif(log has "world"),
LogsWithTest = countif(log has "test")
| project Result = pack_all()
| mv-expand Result
| extend Message = tostring(bag_keys(Result)[0])
| extend Count = tolong(Result[Message])
| project Message, Count
The produced result is:
| Message | Count |
|---------------|-------|
| LogsWithWorld | 2 |
| LogsWithTest | 3 |
|---------------|-------|
My query below seem to be in-efficient, but the Azure "ContainerLog" table contains no Container details, I do a lookup from "KubePodInventory".
let _podInventory = ( KubePodInventory
| where Namespace has "My-k8s-Namespace"
| where ContainerName has_any ('My-Pod' )
| distinct ContainerID , ContainerName, Namespace, Name
);
ContainerLog
| where TimeGenerated between( datetime("2020-06-18 02:00:00 ") .. now())
| join kind=inner _podInventory on $left.ContainerID == $right.ContainerID
| project LogEntry, Name1, Namespace , ContainerName, TimeGenerated
Use the lookup operator, it should be more efficient because the join will be automatically broadcast (distributed).
This worked well for me, combining filterd KubePodInventory with #Avnera's lookup operator seem to be fast.
//# Pieter2020-06 Search for log in Namespace and Container name
let _podInventory = ( KubePodInventory
| where Namespace has "mynamesspace-uat"
| where ContainerName has_any ('pod1' , 'web2' , 'pod3')
| distinct ContainerID, ContainerName, Namespace, PodRestartCount , Name );
ContainerLog
//| where TimeGenerated between( datetime("2020-06-16 02:00:00 ") .. now())
| where TimeGenerated between( datetime("2020-07-13 19:00:00 ") .. datetime("2020-07-15 9:00:00 ") )
| where ( LogEntry has_any( "WARN", "ERRO" ) )
| lookup kind=leftouter (_podInventory) on ContainerID
| project TimeGenerated, Name1, LogEntry, Namespace
I have a JSON file that I want to process with JQ. It has an array of objects inside another object, with a key that I want to use to populate a new array.
In my real use-case this is nested in with a lot of other fluff and there lots more arrays but take this as a simpler but representative example of the kind of thing:
{
"numbers": [
{
"numeral": 1,
"ordinal": "1st",
"word": "One"
},
{
"numeral": 2,
"ordinal": "2nd",
"word": "Two"
},
{
"numeral": 5,
"ordinal": "5th",
"word": "Five"
},
{
"some-other-fluff-i-want-to-ignore": true
}
]
}
I'd like to use JQ to get a new array based on the elements, ignoring some elements and handling the missing ones. e.g.
[
"The 1st word is One",
"The 2nd word is Two",
"Wot no number 3?",
"Wot no number 4?",
"The 5th word is Five"
]
Doing this in a loop for the elements that are there is simple, terse and elegant enough:
.numbers | map( . | select( .numeral) | [ "The", .ordinal, "word is", .word ] | join (" "))
But I can't find a way to cope with the missing entries. I have some code that sort-of works:
.numbers | [
( .[] | select(.numeral == 1) | ( [ "The", .ordinal, "word is", .word ] | join (" ")) ) // "Wot no number 1?",
( .[] | select(.numeral == 2) | ( [ "The", .ordinal, "word is", .word ] | join (" ")) ) // "Wot no number 2?",
( .[] | select(.numeral == 3) | ( [ "The", .ordinal, "word is", .word ] | join (" ")) ) // "Wot no number 3?",
( .[] | select(.numeral == 4) | ( [ "The", .ordinal, "word is", .word ] | join (" ")) ) // "Wot no number 4?",
( .[] | select(.numeral == 5) | ( [ "The", .ordinal, "word is", .word ] | join (" ")) ) // "Wot no number 5?"
]
It produces usable output, after a fashion:
richard#sophia:~$ jq -f make-array.jq < numbers.json
[
"The 1st word is One",
"The 2nd word is Two",
"Wot no number 3?",
"Wot no number 4?",
"The 5th word is Five"
]
richard#sophia:~$
However, whilst it produces the output, handles the missing elements and ignores the bits I don't want, it's obviously extremely naff code that cries out for a for-loop or something similar but I can't see a way in JQ to do this. Any ideas?
jq solution:
jq 'def print(o): "The \(o.ordinal) word is \(o.word)";
.numbers | (reduce map(select(.numeral))[] as $o ({}; .["\($o.numeral)"] = $o)) as $o
| [range(0; ($o | [keys[] | tonumber] | max))
| "\(.+1)" as $i
| if ($o[$i]) then print($o[$i]) else "Wot no number \($i)?" end
]' input.json
The output:
[
"The 1st word is One",
"The 2nd word is Two",
"Wot no number 3?",
"Wot no number 4?",
"The 5th word is Five"
]
Another solution !
jq '[
range(1; ( .numbers | max_by(.numeral)|.numeral ) +1 ) as $range_do_diplay |
.numbers as $thedata | $range_do_diplay |
. as $i |
if ([$thedata[]|contains( { numeral: $i })]|any )
then
($thedata|map(select( .numeral == $i )))|.[0]| "The \(.ordinal) word is \(.word) "
else
"Wot no number \($i)?"
end
] ' numbers.json
This solution use
max_by to find the max value of numeral
range to generate a list o values
use variables to store intermediate value