WebStorm not applying ESLint settings - webstorm

I have the following ESLint configuration in .eslintrc.json file:
{
"extends": [
"eslint:recommended",
"plugin:react-hooks/recommended"
],
"parserOptions": {
"ecmaVersion": 8,
"sourceType": "module",
"ecmaFeatures": {
"jsx": true,
"impliedStrict": true,
"modules": true,
"experimentalObjectRestSpread": true
}
},
"parser": "babel-eslint",
"plugins": [
"react",
"import"
],
"env": {
"browser": true,
"node": true,
"jquery": true,
"es6": true,
"jest": true
},
"globals": {
"jest": true,
"describe": true,
"it": true,
"expect": true,
"before": true,
"after": true,
"beforeEach": true
},
"settings": {
"import/resolver": "webpack"
},
"rules": {
"array-bracket-spacing": [
2,
"never"
],
"array-callback-return": 2,
"arrow-body-style": 0,
"arrow-parens": [
2,
"always"
],
"arrow-spacing": [
2,
{
"before": true,
"after": true
}
],
"block-scoped-var": 2,
"brace-style": [
2,
"1tbs",
{
"allowSingleLine": false
}
],
"capitalized-comments": 0,
"class-methods-use-this": 0,
"comma-dangle": [
2,
"always-multiline"
],
"comma-spacing": [
2,
{
"before": false,
"after": true
}
],
"comma-style": [
2,
"last"
],
"complexity": [
0,
10
],
"computed-property-spacing": [
2,
"never"
],
"consistent-return": 2,
"consistent-this": [
2,
"self"
],
"constructor-super": 2,
"curly": [
2,
"all"
],
"dot-location": [
2,
"object"
],
"dot-notation": 2,
"eqeqeq": [
2,
"smart"
],
"func-call-spacing": [
2,
"never"
],
"func-name-matching": 0,
"func-names": 2,
"func-style": [
2,
"declaration",
{
"allowArrowFunctions": true
}
],
"generator-star-spacing": [
2,
{
"before": false,
"after": true
}
],
"global-require": 2,
"guard-for-in": 2,
"id-blacklist": 0,
"import/no-unresolved": 2,
"import/order": [
"error",
{
"newlines-between": "always-and-inside-groups",
"groups": [
"builtin",
"external",
[
"internal",
"parent"
],
"sibling",
"index"
]
}
],
"indent": [
2
],
"jsx-quotes": [
2,
"prefer-single"
],
"key-spacing": [
2,
{
"beforeColon": false,
"afterColon": true,
"mode": "strict"
}
],
"keyword-spacing": [
2,
{
"before": true,
"after": true,
"overrides": {}
}
],
"line-comment-position": 0,
"linebreak-style": 2,
"lines-around-comment": [
2,
{
"beforeBlockComment": true,
"beforeLineComment": true,
"allowBlockStart": true,
"allowBlockEnd": true
}
],
"max-lines": [
1,
{
"max": 450,
"skipBlankLines": true,
"skipComments": false
}
],
"max-nested-callbacks": [
2,
{
"max": 2
}
],
"max-statements-per-line": [
2,
{
"max": 1
}
],
"multiline-ternary": [
1,
"never"
],
"new-cap": 2,
"new-parens": 2,
"newline-before-return": 0,
"newline-per-chained-call": 0,
"no-alert": 2,
"no-array-constructor": 2,
"no-await-in-loop": 2,
"no-caller": 2,
"no-case-declarations": 2,
"no-class-assign": 2,
"no-compare-neg-zero": 2,
"no-cond-assign": [
2,
"except-parens"
],
"no-confusing-arrow": 2,
"no-console": 2,
"no-const-assign": 2,
"no-constant-condition": 2,
"no-debugger": 2,
"no-div-regex": 2,
"no-dupe-args": 2,
"no-dupe-class-members": 2,
"no-dupe-keys": 2,
"no-duplicate-case": 2,
"no-duplicate-imports": [
2,
{
"includeExports": true
}
],
"no-else-return": 2,
"no-empty": 2,
"no-empty-function": 2,
"no-empty-pattern": 2,
"no-eval": 2,
"no-ex-assign": 2,
"no-extend-native": 2,
"no-extra-bind": 2,
"no-extra-label": 2,
"no-extra-parens": 0,
"no-extra-semi": 2,
"no-fallthrough": 2,
"no-floating-decimal": 2,
"no-func-assign": 2,
"no-global-assign": 2,
"no-implicit-coercion": 2,
"no-implicit-globals": 0,
"no-implied-eval": 2,
"no-inner-declarations": 0,
"no-invalid-regexp": 2,
"no-irregular-whitespace": 2,
"no-iterator": 2,
"no-labels": 2,
"no-lone-blocks": 2,
"no-lonely-if": 2,
"no-loop-func": 2,
"no-magic-numbers": [
0,
{
"ignore": [
-1,
0,
1,
2
],
"enforceConst": true,
"detectObjects": true
}
],
"no-mixed-operators": [
2,
{
"allowSamePrecedence": false
}
],
"no-mixed-spaces-and-tabs": 2,
"no-multi-assign": 2,
"no-multi-spaces": [
2,
{
"exceptions": {
"Property": false
}
}
],
"no-multi-str": 0,
"no-multiple-empty-lines": [
2,
{
"max": 1
}
],
"no-native-reassign": 2,
"no-negated-condition": 2,
"no-nested-ternary": 2,
"no-new": 2,
"no-new-func": 2,
"no-new-object": 2,
"no-new-symbol": 2,
"no-new-wrappers": 2,
"no-octal-escape": 2,
"no-param-reassign": 2,
"no-process-env": 2,
"no-process-exit": 2,
"no-proto": 2,
"no-redeclare": 2,
"no-return-assign": [
2,
"always"
],
"no-return-await": 2,
"no-script-url": 2,
"no-self-assign": [
2,
{
"props": true
}
],
"no-self-compare": 2,
"no-sequences": 2,
"no-shadow": [
2,
{
"hoist": "functions"
}
],
"no-shadow-restricted-names": 2,
"no-spaced-func": 2,
"no-tabs": 0,
"no-template-curly-in-string": 2,
"no-ternary": 0,
"no-this-before-super": 2,
"no-throw-literal": 2,
"no-trailing-spaces": [
2,
{
"skipBlankLines": false
}
],
"no-undef-init": 2,
"no-undefined": 2,
"no-underscore-dangle": 2,
"no-unexpected-multiline": 2,
"no-unmodified-loop-condition": 2,
"no-unneeded-ternary": [
2,
{
"defaultAssignment": false
}
],
"no-unreachable": 2,
"no-unsafe-finally": 2,
"no-unsafe-negation": 2,
"no-unused-expressions": 2,
"no-unused-vars": [
2,
{
"vars": "all",
"args": "after-used"
}
],
"no-use-before-define": [
2,
{
"classes": false,
"functions": false,
"variables": false
}
],
"no-useless-computed-key": 2,
"no-useless-concat": 2,
"no-useless-constructor": 2,
"no-useless-escape": 2,
"no-useless-rename": 2,
"no-useless-return": 2,
"no-var": 0,
"no-void": 2,
"no-warning-comments": 1,
"no-whitespace-before-property": 2,
"no-with": 2,
"object-curly-newline": 0,
"object-curly-spacing": [
2,
"never"
],
"object-property-newline": [
2,
{
"allowMultiplePropertiesPerLine": true
}
],
"object-shorthand": [
2,
"always"
],
"one-var": [
2,
"never"
],
"one-var-declaration-per-line": 0,
"operator-assignment": [
2,
"always"
],
"operator-linebreak": [
2,
"after"
],
"padded-blocks": [
2,
"never"
],
"prefer-arrow-callback": 2,
"prefer-const": 2,
"prefer-destructuring": 0,
"prefer-numeric-literals": 2,
"prefer-promise-reject-errors": 2,
"prefer-rest-params": 2,
"prefer-spread": 2,
"prefer-template": 0,
"quote-props": [
2,
"as-needed"
],
"quotes": [
2,
"single",
"avoid-escape"
],
"radix": 2,
"react/display-name": [
0,
{
"ignoreTranspilerName": false
}
],
"react/forbid-component-props": 0,
"react/forbid-elements": [
2,
{
"forbid": [
"embed"
]
}
],
"react/jsx-boolean-value": [
2,
"always"
],
"react/jsx-closing-bracket-location": [
2,
{
"location": "tag-aligned"
}
],
"react/jsx-curly-spacing": [
2,
"never"
],
"react/jsx-equals-spacing": [
2,
"never"
],
"react/jsx-filename-extension": 2,
"react/jsx-first-prop-new-line": [
2,
"multiline"
],
"react/jsx-handler-names": 0,
"react/jsx-indent": [
2
],
"react/jsx-indent-props": [
2
],
"react/jsx-key": 2,
"react/jsx-max-props-per-line": [
2,
{
"maximum": 1
}
],
"react/jsx-no-bind": 0,
"react/jsx-no-comment-textnodes": 2,
"react/jsx-no-duplicate-props": [
2,
{
"ignoreCase": false
}
],
"react/jsx-no-literals": 2,
"react/jsx-no-target-blank": 2,
"react/jsx-no-undef": 2,
"react/jsx-pascal-case": 2,
"react/jsx-tag-spacing": [
2,
{
"closingSlash": "never",
"beforeSelfClosing": "never",
"afterOpening": "never"
}
],
"react/jsx-uses-react": 2,
"react/jsx-uses-vars": 2,
"react/jsx-wrap-multilines": 2,
"react/no-array-index-key": 1,
"react/no-children-prop": 2,
"react/no-danger": 0,
"react/no-danger-with-children": 2,
"react/no-deprecated": 1,
"react/no-did-mount-set-state": 2,
"react/no-did-update-set-state": 2,
"react/no-direct-mutation-state": 2,
"react/no-find-dom-node": 1,
"react/no-is-mounted": 2,
"react/no-multi-comp": [
2,
{
"ignoreStateless": true
}
],
"react/no-render-return-value": 2,
"react/no-set-state": 0,
"react/no-string-refs": 0,
"react/no-unescaped-entities": 2,
"react/no-unknown-property": 2,
"react/no-unused-prop-types": [
1,
{
"skipShapeProps": true
}
],
"react/prefer-es6-class": 2,
"react/prefer-stateless-function": 2,
"react/prop-types": [
2,
{
"ignore": [
"location",
"history",
"component"
]
}
],
"react/require-default-props": 0,
"react/require-optimization": 1,
"react/require-render-return": 2,
"react/self-closing-comp": 2,
"react/sort-comp": 0,
"react/style-prop-object": 2,
"require-yield": 2,
"rest-spread-spacing": [
2,
"never"
],
"semi": [
2,
"always"
],
"semi-spacing": [
2,
{
"before": false,
"after": true
}
],
"sort-imports": 0,
"sort-keys": 0,
"space-before-blocks": [
2,
"always"
],
"space-before-function-paren": [
2,
{
"anonymous": "never",
"named": "never",
"asyncArrow": "always"
}
],
"space-in-parens": [
2,
"never"
],
"space-infix-ops": 2,
"space-unary-ops": [
2,
{
"words": true,
"nonwords": false
}
],
"symbol-description": 2,
"template-curly-spacing": [
2,
"never"
],
"valid-typeof": [
2,
{
"requireStringLiterals": false
}
],
"vars-on-top": 0,
"wrap-iife": [
2,
"outside"
],
"wrap-regex": 2,
"yoda": [
2,
"never",
{
"exceptRange": false,
"onlyEquality": false
}
]
},
"overrides": [
{
"files": ["**/*.tsx", "**/*.ts"],
"extends": "plugin:#typescript-eslint/recommended",
"rules": {
"#typescript-eslint/ban-ts-ignore": 0,
"#typescript-eslint/ban-types": 1,
"#typescript-eslint/ban-ts-comment": 0,
"#typescript-eslint/no-var-requires": 0,
"#typescript-eslint/prefer-interface": 0,
"#typescript-eslint/explicit-function-return-type": 0,
"#typescript-eslint/explicit-module-boundary-types": 0,
"#typescript-eslint/indent": 2,
"#typescript-eslint/no-use-before-define": [
2,
{
"classes": false,
"functions": false,
"variables": false
}
],
"react/jsx-filename-extension": [
1,
{
"extensions": [".jsx", ".tsx"]
}
]
}
}
]
}
In the WebStorm's ESLint configuration, I set the config file to .eslintrc.json. In this, I have made the indents to be of size 2.
But I get an error that it expects indent of size 4. I have invalidated caches and restarted several times, still the same issue. How should I solve this?

Related

CoC not sending autocomplete request message to language server

My nvim autocomplete window is not coming up when editing C++ code.
I've been trying to follow this debugging guideline but I haven't had much success. I'm using the Kythe language server configured as follows in coc-settings.json:
{
"languageserver": {
"kythe": {
"command": "/full/path/omitted/kythe_languageserver",
"filetypes": ["python", "go", "java", "cpp", "cc", "c++", "proto"],
"trace.server" : "verbose"
}
}
}
When opening a C++ file and running :CocList services, I can see that the language server has started:
languageserver.kythe [running] python, go, java, cpp, cc, c++, proto
And indeed in the :CocCommand workspace.showOutput, there are messages indicating a successful initialization:
[Trace - 4:45:05 PM] Received response 'initialize - (0)' in 264ms.
Result: {
"capabilities": {
"textDocumentSync": 1,
"hoverProvider": true,
"definitionProvider": true,
"referencesProvider": true
}
}
One thought, is there a missing capability here that is required for autocomplete to come up? In any case, when I edit the file and try to, for example, type std:: expecting some form of autocompletion for that namespace, nothing happens and the only messages sent to the language server seem to be as follows:
[Trace - 4:45:47 PM] Sending notification 'textDocument/didChange'.
Params: {
"textDocument": {
"uri": "[redacted]",
"version": 4
},
"contentChanges": [
{
"text": "[redacted]"
}
]
}
From my limited knowledge, I'm pretty sure that autocompletion also requires a different message type to be sent to the language server, no? textDocument/didChange seems to be only for updating the state.
Edit: Full set of requested capabilities by nvim:
"capabilities": {
"workspace": {
"applyEdit": true,
"workspaceEdit": {
"documentChanges": true,
"resourceOperations": [
"create",
"rename",
"delete"
],
"failureHandling": "textOnlyTransactional"
},
"didChangeConfiguration": {
"dynamicRegistration": true
},
"didChangeWatchedFiles": {
"dynamicRegistration": true
},
"symbol": {
"dynamicRegistration": true,
"symbolKind": {
"valueSet": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26
]
},
"tagSupport": {
"valueSet": [
1
]
}
},
"executeCommand": {
"dynamicRegistration": true
},
"configuration": true,
"workspaceFolders": true
},
"textDocument": {
"publishDiagnostics": {
"relatedInformation": true,
"versionSupport": false,
"tagSupport": {
"valueSet": [
1,
2
]
}
},
"synchronization": {
"dynamicRegistration": true,
"willSave": true,
"willSaveWaitUntil": true,
"didSave": true
},
"completion": {
"dynamicRegistration": true,
"contextSupport": true,
"completionItem": {
"snippetSupport": true,
"commitCharactersSupport": true,
"documentationFormat": [
"markdown",
"plaintext"
],
"deprecatedSupport": true,
"preselectSupport": true
},
"completionItemKind": {
"valueSet": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25
]
}
},
"hover": {
"dynamicRegistration": true,
"contentFormat": [
"markdown",
"plaintext"
]
},
"signatureHelp": {
"dynamicRegistration": true,
"signatureInformation": {
"documentationFormat": [
"markdown",
"plaintext"
],
"parameterInformation": {
"labelOffsetSupport": true
}
}
},
"definition": {
"dynamicRegistration": true
},
"references": {
"dynamicRegistration": true
},
"documentHighlight": {
"dynamicRegistration": true
},
"documentSymbol": {
"dynamicRegistration": true,
"symbolKind": {
"valueSet": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26
]
},
"hierarchicalDocumentSymbolSupport": true,
"tagSupport": {
"valueSet": [
1
]
}
},
"codeAction": {
"dynamicRegistration": true,
"isPreferredSupport": true,
"codeActionLiteralSupport": {
"codeActionKind": {
"valueSet": [
"",
"quickfix",
"refactor",
"refactor.extract",
"refactor.inline",
"refactor.rewrite",
"source",
"source.organizeImports"
]
}
}
},
"codeLens": {
"dynamicRegistration": true
},
"formatting": {
"dynamicRegistration": true
},
"rangeFormatting": {
"dynamicRegistration": true
},
"onTypeFormatting": {
"dynamicRegistration": true
},
"rename": {
"dynamicRegistration": true,
"prepareSupport": true
},
"documentLink": {
"dynamicRegistration": true,
"tooltipSupport": true
},
"typeDefinition": {
"dynamicRegistration": true
},
"implementation": {
"dynamicRegistration": true
},
"declaration": {
"dynamicRegistration": true
},
"colorProvider": {
"dynamicRegistration": true
},
"foldingRange": {
"dynamicRegistration": true,
"rangeLimit": 5000,
"lineFoldingOnly": true
},
"selectionRange": {
"dynamicRegistration": true
}
},
"window": {
"workDoneProgress": true
}
},
Kythe language server does not support auto complete capabilities as of yet.

Set segment duration with media convert

With AWS media convert everytime I convert an mp4 (6 second video) to a dash, the segment duration is about 30 seconds however I'd much appreciate if it were 1 second or less:
<SegmentTemplate timescale="90000" duration="324000" startNumber="1"/>
<Representation id="1" width="1280" height="720" bandwidth="72000000" codecs="avc1.4d4032">
<SegmentTemplate media="5f8283b60a3ac3640191892_$Number%09d$.mp4" initialization="5f8283b60a3ac3640191892init.mp4" duration="324000" startNumber="1"/>
</Representation>
<Representation id="2" width="1920" height="1080" bandwidth="16200000" codecs="avc1.4d4029">
<SegmentTemplate media="5f8283b60a3ac3640191891_$Number%09d$.mp4" initialization="5f8283b60a3ac3640191891init.mp4" duration="324000" startNumber="1"/>
</Representation>
</AdaptationSet>
I've tried messing with the job settings, but nothing I did seemed to work. Sometimes I got it down to 10 seconds, but that still isn't great, anyway here are the settings I used:
$jobSetting = [
"OutputGroups"=> [
[
"CustomName"=> "nicenice",
"Name"=> "DASH ISO",
"Outputs"=> [
[
"ContainerSettings"=> [
"Container"=> "MPD"
],
"VideoDescription"=> [
"Width"=> 1920,
"ScalingBehavior"=> "DEFAULT",
"Height"=> 1080,
"TimecodeInsertion"=> "DISABLED",
"AntiAlias"=> "ENABLED",
"Sharpness"=> 50,
"CodecSettings"=> [
"Codec"=> "H_264",
"H264Settings"=> [
"InterlaceMode"=> "PROGRESSIVE",
"NumberReferenceFrames"=> 3,
"Syntax"=> "DEFAULT",
"Softness"=> 0,
"GopClosedCadence"=> 1,
"GopSize"=> 60,
"Slices"=> 1,
"GopBReference"=> "DISABLED",
"SlowPal"=> "DISABLED",
"SpatialAdaptiveQuantization"=> "ENABLED",
"TemporalAdaptiveQuantization"=> "ENABLED",
"FlickerAdaptiveQuantization"=> "DISABLED",
"EntropyEncoding"=> "CABAC",
"Bitrate"=> 16200000,
"FramerateControl"=> "INITIALIZE_FROM_SOURCE",
"RateControlMode"=> "CBR",
"CodecProfile"=> "MAIN",
"Telecine"=> "NONE",
"MinIInterval"=> 0,
"AdaptiveQuantization"=> "HIGH",
"CodecLevel"=> "AUTO",
"FieldEncoding"=> "PAFF",
"SceneChangeDetect"=> "ENABLED",
"QualityTuningLevel"=> "SINGLE_PASS",
"FramerateConversionAlgorithm"=> "DUPLICATE_DROP",
"UnregisteredSeiTimecode"=> "DISABLED",
"GopSizeUnits"=> "FRAMES",
"ParControl"=> "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames"=> 2,
"RepeatPps"=> "DISABLED",
"DynamicSubGop"=> "STATIC"
]
],
"AfdSignaling"=> "NONE",
"DropFrameTimecode"=> "ENABLED",
"RespondToAfd"=> "NONE",
"ColorMetadata"=> "INSERT"
],
"NameModifier"=> "1"
],
[
"ContainerSettings"=> [
"Container"=> "MPD"
],
"VideoDescription"=> [
"Width"=> 1280,
"ScalingBehavior"=> "DEFAULT",
"Height"=> 720,
"TimecodeInsertion"=> "DISABLED",
"AntiAlias"=> "ENABLED",
"Sharpness"=> 50,
"CodecSettings"=> [
"Codec"=> "H_264",
"H264Settings"=> [
"InterlaceMode"=> "PROGRESSIVE",
"NumberReferenceFrames"=> 3,
"Syntax"=> "DEFAULT",
"Softness"=> 0,
"GopClosedCadence"=> 1,
"GopSize"=> 60,
"Slices"=> 1,
"GopBReference"=> "DISABLED",
"SlowPal"=> "DISABLED",
"SpatialAdaptiveQuantization"=> "ENABLED",
"TemporalAdaptiveQuantization"=> "ENABLED",
"FlickerAdaptiveQuantization"=> "DISABLED",
"EntropyEncoding"=> "CABAC",
"Bitrate"=> 7200000,
"FramerateControl"=> "INITIALIZE_FROM_SOURCE",
"RateControlMode"=> "CBR",
"CodecProfile"=> "MAIN",
"Telecine"=> "NONE",
"MinIInterval"=> 0,
"AdaptiveQuantization"=> "HIGH",
"CodecLevel"=> "AUTO",
"FieldEncoding"=> "PAFF",
"SceneChangeDetect"=> "ENABLED",
"QualityTuningLevel"=> "SINGLE_PASS",
"FramerateConversionAlgorithm"=> "DUPLICATE_DROP",
"UnregisteredSeiTimecode"=> "DISABLED",
"GopSizeUnits"=> "FRAMES",
"ParControl"=> "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames"=> 2,
"RepeatPps"=> "DISABLED",
"DynamicSubGop"=> "STATIC"
]
],
"AfdSignaling"=> "NONE",
"DropFrameTimecode"=> "ENABLED",
"RespondToAfd"=> "NONE",
"ColorMetadata"=> "INSERT"
],
"NameModifier"=> "2"
],
[
"ContainerSettings"=> [
"Container"=> "MPD"
],
"AudioDescriptions"=> [
[
"AudioTypeControl"=> "FOLLOW_INPUT",
"AudioSourceName"=> "Audio Selector 1",
"CodecSettings"=> [
"Codec"=> "AAC",
"AacSettings"=> [
"AudioDescriptionBroadcasterMix"=> "NORMAL",
"Bitrate"=> 96000,
"RateControlMode"=> "CBR",
"CodecProfile"=> "LC",
"CodingMode"=> "CODING_MODE_2_0",
"RawFormat"=> "NONE",
"SampleRate"=> 48000,
"Specification"=> "MPEG4"
]
],
"LanguageCodeControl"=> "FOLLOW_INPUT"
]
],
"NameModifier"=> "3"
]
],
"OutputGroupSettings"=> [
"Type"=> "DASH_ISO_GROUP_SETTINGS",
"DashIsoGroupSettings"=> [
"SegmentLength"=> 1,
"Destination"=> "s3://cactustestphp/videouploads/".$link . "/".$link,
"FragmentLength"=> 2,
"SegmentControl"=> "SEGMENTED_FILES",
"MpdProfile"=> "MAIN_PROFILE",
"HbbtvCompliance"=> "NONE"
]
]
]
],
"AdAvailOffset"=> 0,
"Inputs"=> [
[
"AudioSelectors"=> [
"Audio Selector 1"=> [
"Offset"=> 0,
"DefaultSelection"=> "DEFAULT",
"ProgramSelection"=> 1
]
],
"VideoSelector"=> [
"ColorSpace"=> "FOLLOW",
"Rotate"=> "DEGREE_0",
"AlphaBehavior"=> "DISCARD"
],
"FilterEnable"=> "AUTO",
"PsiControl"=> "USE_PSI",
"FilterStrength"=> 0,
"DeblockFilter"=> "DISABLED",
"DenoiseFilter"=> "DISABLED",
"InputScanType"=> "AUTO",
"TimecodeSource"=> "ZEROBASED",
"FileInput"=> "s3://cactustestphp/videouploads/test/". $fileid
]
]
];
Json:
{
"Queue": "!!",
"UserMetadata": {
"Customer": "Amazon"
},
"Role": "!!",
"Settings": {
"OutputGroups": [
{
"CustomName": "nicenice",
"Name": "DASH ISO",
"Outputs": [
{
"ContainerSettings": {
"Container": "MPD"
},
"VideoDescription": {
"Width": 3840,
"ScalingBehavior": "DEFAULT",
"Height": 2160,
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"FramerateDenominator": 1,
"GopClosedCadence": 1,
"GopSize": 30,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"SpatialAdaptiveQuantization": "ENABLED",
"TemporalAdaptiveQuantization": "ENABLED",
"FlickerAdaptiveQuantization": "DISABLED",
"EntropyEncoding": "CABAC",
"Bitrate": 66200000,
"FramerateControl": "SPECIFIED",
"RateControlMode": "CBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"FramerateNumerator": 30,
"MinIInterval": 0,
"AdaptiveQuantization": "HIGH",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "SINGLE_PASS",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED",
"DynamicSubGop": "STATIC"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"NameModifier": "1"
},
{
"ContainerSettings": {
"Container": "MPD"
},
"VideoDescription": {
"Width": 1920,
"ScalingBehavior": "DEFAULT",
"Height": 1080,
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"FramerateDenominator": 1,
"GopClosedCadence": 1,
"GopSize": 30,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"SpatialAdaptiveQuantization": "ENABLED",
"TemporalAdaptiveQuantization": "ENABLED",
"FlickerAdaptiveQuantization": "DISABLED",
"EntropyEncoding": "CABAC",
"Bitrate": 16200000,
"FramerateControl": "SPECIFIED",
"RateControlMode": "CBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"FramerateNumerator": 30,
"MinIInterval": 0,
"AdaptiveQuantization": "HIGH",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "SINGLE_PASS",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED",
"DynamicSubGop": "STATIC"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"NameModifier": "2"
},
{
"ContainerSettings": {
"Container": "MPD"
},
"VideoDescription": {
"Width": 1280,
"ScalingBehavior": "DEFAULT",
"Height": 720,
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"FramerateDenominator": 1,
"GopClosedCadence": 1,
"GopSize": 30,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"SpatialAdaptiveQuantization": "ENABLED",
"TemporalAdaptiveQuantization": "ENABLED",
"FlickerAdaptiveQuantization": "DISABLED",
"EntropyEncoding": "CABAC",
"Bitrate": 5200000,
"FramerateControl": "SPECIFIED",
"RateControlMode": "CBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"FramerateNumerator": 30,
"MinIInterval": 0,
"AdaptiveQuantization": "HIGH",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "SINGLE_PASS",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED",
"DynamicSubGop": "STATIC"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"NameModifier": "3"
},
{
"ContainerSettings": {
"Container": "MPD"
},
"VideoDescription": {
"Width": 640,
"ScalingBehavior": "DEFAULT",
"Height": 360,
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"FramerateDenominator": 1,
"GopClosedCadence": 1,
"GopSize": 30,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"SpatialAdaptiveQuantization": "ENABLED",
"TemporalAdaptiveQuantization": "ENABLED",
"FlickerAdaptiveQuantization": "DISABLED",
"EntropyEncoding": "CABAC",
"Bitrate": 1200000,
"FramerateControl": "SPECIFIED",
"RateControlMode": "CBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"FramerateNumerator": 30,
"MinIInterval": 0,
"AdaptiveQuantization": "HIGH",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "SINGLE_PASS",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED",
"DynamicSubGop": "STATIC"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"NameModifier": "4"
},
{
"ContainerSettings": {
"Container": "MPD"
},
"VideoDescription": {
"Width": 256,
"ScalingBehavior": "DEFAULT",
"Height": 144,
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"FramerateDenominator": 1,
"GopClosedCadence": 1,
"GopSize": 30,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"SpatialAdaptiveQuantization": "ENABLED",
"TemporalAdaptiveQuantization": "ENABLED",
"FlickerAdaptiveQuantization": "DISABLED",
"EntropyEncoding": "CABAC",
"Bitrate": 200000,
"FramerateControl": "SPECIFIED",
"RateControlMode": "CBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"FramerateNumerator": 30,
"MinIInterval": 0,
"AdaptiveQuantization": "HIGH",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "SINGLE_PASS",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED",
"DynamicSubGop": "STATIC"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"NameModifier": "5"
},
{
"ContainerSettings": {
"Container": "MPD"
},
"AudioDescriptions": [
{
"AudioTypeControl": "FOLLOW_INPUT",
"AudioSourceName": "Audio Selector 1",
"CodecSettings": {
"Codec": "AAC",
"AacSettings": {
"AudioDescriptionBroadcasterMix": "NORMAL",
"Bitrate": 96000,
"RateControlMode": "CBR",
"CodecProfile": "LC",
"CodingMode": "CODING_MODE_2_0",
"RawFormat": "NONE",
"SampleRate": 48000,
"Specification": "MPEG4"
}
},
"LanguageCodeControl": "FOLLOW_INPUT"
}
],
"NameModifier": "6"
}
],
"OutputGroupSettings": {
"Type": "DASH_ISO_GROUP_SETTINGS",
"DashIsoGroupSettings": {
"SegmentLength": 1,
"Destination": "!!/videouploads/5fa1ababa7cea975176544/5fa1ababa7cea975176544",
"FragmentLength": 1,
"SegmentControl": "SEGMENTED_FILES",
"MpdProfile": "MAIN_PROFILE",
"HbbtvCompliance": "NONE"
}
}
}
],
"AdAvailOffset": 0,
"Inputs": [
{
"AudioSelectors": {
"Audio Selector 1": {
"Offset": 0,
"DefaultSelection": "DEFAULT",
"ProgramSelection": 1
}
},
"VideoSelector": {
"ColorSpace": "FOLLOW",
"Rotate": "DEGREE_0",
"AlphaBehavior": "DISCARD"
},
"FilterEnable": "AUTO",
"PsiControl": "USE_PSI",
"FilterStrength": 0,
"DeblockFilter": "DISABLED",
"DenoiseFilter": "DISABLED",
"TimecodeSource": "ZEROBASED",
"FileInput": "!!/videouploads/test/5fa1ababa7cea975176544.mp4"
}
]
},
"AccelerationSettings": {
"Mode": "DISABLED"
},
"StatusUpdateInterval": "SECONDS_60",
"Priority": 0
}
So, in other words, what would I have to change to make my segment duration set to 1 second or less?
Thank you for providing the job's JSON settings for further review. Looking at the OutputGroupSettings I can see that you are specifying a SegmentLength value of 1 second so the resulting fragmented MP4 files should be about 1 second in length give or take a few frames.
How are you confirming that the resulting fMP4 files are 30 and 10 seconds respectively? The most accurate way of determining this information is to combine a variant's init MP4 with one of the fragments into a separate file and observe it using a media inspector like Mediainfo or ffprobe. I used your job settings on an MP4 file I had available to me and confirmed that the resulting MP4 segments produces are 1 second in length:
Concatenate a segment with its init file for the full MP4 asset
$ cat 5fa1ababa7cea9751765441init.mp4 >> 5fa1ababa7cea9751765441_concat.mp4 && cat 5fa1ababa7cea9751765441_000000001.mp4 >> 5fa1ababa7cea9751765441_concat.mp4
Probe the newly concatenated file to review the details
$ ffprobe -hide_banner -i 5fa1ababa7cea9751765441_concat.mp4
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from '5fa1ababa7cea9751765441_concat.mp4':
Metadata:
major_brand : isom
minor_version : 1
compatible_brands: isomavc1dash
creation_time : 2020-11-20T20:30:59.000000Z
Duration: 00:00:01.03, start: 0.066667, bitrate: 7658 kb/s
Stream #0:0(und): Video: h264 (Main) (avc1 / 0x31637661), yuv420p, 3840x2160 [SAR 1:1 DAR 16:9], 7648 kb/s, 30 fps, 30 tbr, 90k tbn, 60 tbc (default)
Metadata:
creation_time : 2020-11-20T20:30:59.000000Z
handler_name : ETI ISO Video Media Handler
encoder : Elemental H.264
I'm curious how you are confirming the resulting duration of each fMP4 asset and if you can try the above steps as well if you have not already done so.

using mongodb case insentive regex with case insentive index

is mongo regex ignoring my index? I have a case insentive index, but by the look of things my regex search recognize it and ignores it.
db.getCollection("myCol").find({ value: /^mysearchVal/i }}).explain(...)
I have 95, 708 docs total.
output:
{
"queryPlanner": {
"plannerVersion": 1,
"namespace": "myDb.myCol",
"indexFilterSet": false,
"parsedQuery": {
"Value": {
"$regex": "^mysearchVal",
"$options": "i"
}
},
"winningPlan": {
"stage": "FETCH",
"filter": {
"Value": {
"$regex": "^mysearchVal",
"$options": "i"
}
},
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"Value": 1
},
"indexName": "value_case_insensitive_and_unique",
"collation": {
"locale": "en",
"caseLevel": false,
"caseFirst": "off",
"strength": 2,
"numericOrdering": false,
"alternate": "non-ignorable",
"maxVariable": "punct",
"normalization": false,
"backwards": false,
"version": "57.1"
},
"isMultiKey": false,
"multiKeyPaths": {
"Value": []
},
"isUnique": true,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"Value": [
"[\"\", {})",
"[/^mysearchVal/i, /^mysearchVal/i]"
]
}
}
},
"rejectedPlans": []
},
"executionStats": {
"executionSuccess": true,
"nReturned": 1,
"executionTimeMillis": 1447,
"totalKeysExamined": 95708,
"totalDocsExamined": 95708,
"executionStages": {
"stage": "FETCH",
"filter": {
"Value": {
"$regex": "^mysearchVal",
"$options": "i"
}
},
"nReturned": 1,
"executionTimeMillisEstimate": 1270,
"works": 95709,
"advanced": 1,
"needTime": 95707,
"needYield": 0,
"saveState": 785,
"restoreState": 785,
"isEOF": 1,
"invalidates": 0,
"docsExamined": 95708,
"alreadyHasObj": 0,
"inputStage": {
"stage": "IXSCAN",
"nReturned": 95708,
"executionTimeMillisEstimate": 596,
"works": 95709,
"advanced": 95708,
"needTime": 0,
"needYield": 0,
"saveState": 785,
"restoreState": 785,
"isEOF": 1,
"invalidates": 0,
"keyPattern": {
"Value": 1
},
"indexName": "value_case_insensitive_and_unique",
"collation": {
"locale": "en",
"caseLevel": false,
"caseFirst": "off",
"strength": 2,
"numericOrdering": false,
"alternate": "non-ignorable",
"maxVariable": "punct",
"normalization": false,
"backwards": false,
"version": "57.1"
},
"isMultiKey": false,
"multiKeyPaths": {
"Value": []
},
"isUnique": true,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"Value": [
"[\"\", {})",
"[/^mysearchVal/i, /^mysearchVal/i]"
]
},
"keysExamined": 95708,
"seeks": 1,
"dupsTested": 0,
"dupsDropped": 0,
"seenInvalidated": 0
}
},
"allPlansExecution": []
},
"ok": 1.0
}
the output shows 95,708 keys and docs examined, 1 doc returned. really? did the index apply in this case or am I missing a point or two?
Case insensitive regular expression queries generally cannot use
indexes effectively. The $regex implementation is not collation-aware
and is unable to utilize case-insensitive indexes.
https://docs.mongodb.com/manual/reference/operator/query/regex/#index-use

Spark streaming to Elasticsearch from Cassandra and MySQL correct configuration

I have inherited a cluster of mess and trying to figure out if this is the correct way to setup spark.
Goal: we have approx 13M records in Mysql across several tables that needs to be index in ElasticSearch. We have similarly have several million records in Cassandra that needs to be index in Elasticsearch also.
We do a total reindexation because a lot data gets changed in the DB and currently the application doesn't crud to ES.
All setup are on AWS Ec2 clusters with some RDS for MYSQL.
I have 4 EC2 (m4.xlarge) machines running ElasticSearch Nodes.
I have 3 EC2 machines running Cassandra Nodes
I have 1 Master + 1 Slave RDS running MySQL.
I have 1 C3.XLarge EC2 with Spark setup (everything is actually default configuration).
The scala application that was developed takes the mysql data, runs a mapping and transform them into RDD and push to the ES. Same goes for Cassandra.
Issue:
CPU + I/O taps out the ElasticSearch machines and I am at 100%. The entire process takes several hours.
Before I optimize code - should i be running the Spark system on standalone or should I be using it differently and why (so i can tell the IT I need to deploy or change the servers)
Thanks
UPDATE
{
"timestamp": 1470397729366,
"name": "es-server-1",
"transport_address": "inet[/XXX.xx.xx.XX:9300]",
"host": "es-server-1",
"ip": [
"inet[/xxx.xx.xx.XX:9300]",
"NONE"
],
"indices": {
"docs": {
"count": 13297950,
"deleted": 4132597
},
"store": {
"size": "19gb",
"size_in_bytes": 20405153604,
"throttle_time": "8.5h",
"throttle_time_in_millis": 30887045
},
"indexing": {
"index_total": 1306040463,
"index_time": "39.4d",
"index_time_in_millis": 3409319879,
"index_current": 0,
"delete_total": 3176,
"delete_time": "468ms",
"delete_time_in_millis": 468,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time": "3.7h",
"throttle_time_in_millis": 13387134
},
"get": {
"total": 0,
"get_time": "0s",
"time_in_millis": 0,
"exists_total": 0,
"exists_time": "0s",
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time": "0s",
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 2,
"query_total": 91817250,
"query_time": "7.7d",
"query_time_in_millis": 669964087,
"query_current": 0,
"fetch_total": 6111935,
"fetch_time": "26.8m",
"fetch_time_in_millis": 1609543,
"fetch_current": 0
},
"merges": {
"current": 1,
"current_docs": 140211,
"current_size": "358.5mb",
"current_size_in_bytes": 375921439,
"total": 1308682,
"total_time": "52.2d",
"total_time_in_millis": 4515401497,
"total_docs": 22683582490,
"total_size": "18.3tb",
"total_size_in_bytes": 20196592980131
},
"refresh": {
"total": 6513707,
"total_time": "5.9d",
"total_time_in_millis": 515651065
},
"flush": {
"total": 49408,
"total_time": "1.7d",
"total_time_in_millis": 147150501
},
"warmer": {
"current": 0,
"total": 1696226,
"total_time": "26.4m",
"total_time_in_millis": 1589979
},
"filter_cache": {
"memory_size": "429.2mb",
"memory_size_in_bytes": 450131652,
"evictions": 1767369
},
"id_cache": {
"memory_size": "0b",
"memory_size_in_bytes": 0
},
"fielddata": {
"memory_size": "411mb",
"memory_size_in_bytes": 431001032,
"evictions": 0
},
"percolate": {
"total": 0,
"get_time": "0s",
"time_in_millis": 0,
"current": 0,
"memory_size_in_bytes": -1,
"memory_size": "-1b",
"queries": 0
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 272,
"memory": "113.4mb",
"memory_in_bytes": 118989720,
"index_writer_memory": "0b",
"index_writer_memory_in_bytes": 0,
"index_writer_max_memory": "813.7mb",
"index_writer_max_memory_in_bytes": 853250866,
"version_map_memory": "0b",
"version_map_memory_in_bytes": 0,
"fixed_bit_set": "32.1mb",
"fixed_bit_set_memory_in_bytes": 33692696
},
"translog": {
"operations": 0,
"size": "17b",
"size_in_bytes": 17
},
"suggest": {
"total": 4,
"time": "0s",
"time_in_millis": 0,
"current": 0
},
"query_cache": {
"memory_size": "0b",
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
}
},
"os": {
"timestamp": 1470397728691,
"uptime": "4.2h",
"uptime_in_millis": 15430550,
"load_average": [
0.01,
0.13,
0.17
],
"cpu": {
"sys": 0,
"user": 0,
"idle": 98,
"usage": 0,
"stolen": 0
},
"mem": {
"free": "1.5gb",
"free_in_bytes": 1627320320,
"used": "14.1gb",
"used_in_bytes": 15200206848,
"free_percent": 42,
"used_percent": 57,
"actual_free": "6.6gb",
"actual_free_in_bytes": 7102046208,
"actual_used": "9gb",
"actual_used_in_bytes": 9725480960
},
"swap": {
"used": "0b",
"used_in_bytes": 0,
"free": "0b",
"free_in_bytes": 0
}
},
"process": {
"timestamp": 1470397729391,
"open_file_descriptors": 1072,
"cpu": {
"percent": 8,
"sys": "2.8d",
"sys_in_millis": 247968490,
"user": "55.5d",
"user_in_millis": 4803132120,
"total": "58.4d",
"total_in_millis": 5051100610
},
"mem": {
"resident": "8.8gb",
"resident_in_bytes": 9499480064,
"share": "490.4mb",
"share_in_bytes": 514248704,
"total_virtual": "13gb",
"total_virtual_in_bytes": 14003888128
}
},
"jvm": {
"timestamp": 1470397728691,
"uptime": "168.6d",
"uptime_in_millis": 14575120452,
"mem": {
"heap_used": "4gb",
"heap_used_in_bytes": 4337667264,
"heap_used_percent": 51,
"heap_committed": "7.8gb",
"heap_committed_in_bytes": 8378908672,
"heap_max": "7.8gb",
"heap_max_in_bytes": 8378908672,
"non_heap_used": "90.5mb",
"non_heap_used_in_bytes": 94943920,
"non_heap_committed": "112mb",
"non_heap_committed_in_bytes": 117506048,
"pools": {
"young": {
"used": "263mb",
"used_in_bytes": 275871008,
"max": "266.2mb",
"max_in_bytes": 279183360,
"peak_used": "266.2mb",
"peak_used_in_bytes": 279183360,
"peak_max": "266.2mb",
"peak_max_in_bytes": 279183360
},
"survivor": {
"used": "513.3kb",
"used_in_bytes": 525696,
"max": "33.2mb",
"max_in_bytes": 34865152,
"peak_used": "33.2mb",
"peak_used_in_bytes": 34865152,
"peak_max": "33.2mb",
"peak_max_in_bytes": 34865152
},
"old": {
"used": "3.7gb",
"used_in_bytes": 4061270560,
"max": "7.5gb",
"max_in_bytes": 8064860160,
"peak_used": "5.8gb",
"peak_used_in_bytes": 6271548128,
"peak_max": "7.5gb",
"peak_max_in_bytes": 8064860160
}
}
},
"threads": {
"count": 75,
"peak_count": 90
},
"gc": {
"collectors": {
"young": {
"collection_count": 2827443,
"collection_time": "22.1h",
"collection_time_in_millis": 79772838
},
"old": {
"collection_count": 2420,
"collection_time": "4.2m",
"collection_time_in_millis": 253792
}
}
},
"buffer_pools": {
"direct": {
"count": 3519,
"used": "82.1mb",
"used_in_bytes": 86127850,
"total_capacity": "82.1mb",
"total_capacity_in_bytes": 86127850
},
"mapped": {
"count": 250,
"used": "2.8gb",
"used_in_bytes": 3045093527,
"total_capacity": "2.8gb",
"total_capacity_in_bytes": 3045093527
}
}
},
"thread_pool": {
"generic": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 16062931
},
"index": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 35253
},
"bench": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"get": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"merge": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 3112788
},
"suggest": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 4
},
"bulk": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 46447866
},
"optimize": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 159210301
},
"flush": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 756948
},
"search": {
"threads": 12,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 12,
"completed": 133987209
},
"listener": {
"threads": 2,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 26868988
},
"percolate": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 227031495
},
"refresh": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 2,
"completed": 6532756
}
},
"network": {
"tcp": {
"active_opens": 1782589,
"passive_opens": 20362132,
"curr_estab": 226,
"in_segs": 2535219205,
"out_segs": 2254643744,
"retrans_segs": 227538,
"estab_resets": 273558,
"attempt_fails": 14303,
"in_errs": 100,
"out_rsts": 260243
}
},
"fs": {
"timestamp": 1470397728691,
"total": {
"total": "196.6gb",
"total_in_bytes": 211110395904,
"free": "177.4gb",
"free_in_bytes": 190571290624,
"available": "167.4gb",
"available_in_bytes": 179800317952,
"disk_reads": 567609070,
"disk_writes": 216561071,
"disk_io_op": 784170141,
"disk_read_size": "15.6tb",
"disk_read_size_in_bytes": 17215484946432,
"disk_write_size": "23.2tb",
"disk_write_size_in_bytes": 25537856344064,
"disk_io_size": "38.8tb",
"disk_io_size_in_bytes": 42753341290496,
"disk_queue": "0",
"disk_service_time": "0"
},
"data": [
{
"path": "/data/elasticsearch/data0/es-server-1/nodes/0",
"mount": "/data/elasticsearch/data0",
"dev": "/dev/xvdf",
"total": "98.3gb",
"total_in_bytes": 105555197952,
"free": "88.8gb",
"free_in_bytes": 95364030464,
"available": "83.7gb",
"available_in_bytes": 89978544128,
"disk_reads": 280044788,
"disk_writes": 107670720,
"disk_io_op": 387715508,
"disk_read_size": "7.7tb",
"disk_read_size_in_bytes": 8567588643840,
"disk_write_size": "11.5tb",
"disk_write_size_in_bytes": 12689373003776,
"disk_io_size": "19.3tb",
"disk_io_size_in_bytes": 21256961647616,
"disk_queue": "0",
"disk_service_time": "0"
},
{
"path": "/data/elasticsearch/data1/es-server-1/nodes/0",
"mount": "/data/elasticsearch/data1",
"dev": "/dev/xvdg",
"total": "98.3gb",
"total_in_bytes": 105555197952,
"free": "88.6gb",
"free_in_bytes": 95207260160,
"available": "83.6gb",
"available_in_bytes": 89821773824,
"disk_reads": 287564282,
"disk_writes": 108890351,
"disk_io_op": 396454633,
"disk_read_size": "7.8tb",
"disk_read_size_in_bytes": 8647896302592,
"disk_write_size": "11.6tb",
"disk_write_size_in_bytes": 12848483340288,
"disk_io_size": "19.5tb",
"disk_io_size_in_bytes": 21496379642880,
"disk_queue": "0",
"disk_service_time": "0"
}
]
},
"transport": {
"server_open": 104,
"rx_count": 883258049,
"rx_size": "1.8tb",
"rx_size_in_bytes": 2051353458225,
"tx_count": 682553258,
"tx_size": "1.6tb",
"tx_size_in_bytes": 1804963466280
},
"http": {
"current_open": 12,
"total_opened": 19973834
},
"breakers": {
"request": {
"limit_size_in_bytes": 3351563468,
"limit_size": "3.1gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 5027345203,
"limit_size": "4.6gb",
"estimated_size_in_bytes": 431001032,
"estimated_size": "411mb",
"overhead": 1.03,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 5865236070,
"limit_size": "5.4gb",
"estimated_size_in_bytes": 431001032,
"estimated_size": "411mb",
"overhead": 1,
"tripped": 0
}
}
}

how to apply conditions on JSON response in Jmeter?

Response is as follows,
I have to extract all 'deviceResponseStatus.id' where status="PENDING"
how to implement it using JSON Path or REGEX in Jmeter?
{
"apiResponseStatus": "SUCCESS",
"deviceResponseList": [
{
"apiResponseStatus": "SUCCESS",
"id": 23,
"ownership": "CORPORATE",
"deviceName": "Demimbu",
"deviceType": "MOBILE",
"osType": "ANDROID",
"inTotalStorage": 0,
"inAvailableStorage": 0,
"exTotalStorage": 0,
"exAvailableStorage": 0,
"osVersion": "4.2",
"buildVersion": "1.01.08",
"status": "PENDING",
"isDeleted": false,
"policyIdNames": {
"id": 3,
"name": "Test Policy 3"
},
"deviceGroupIdNames": {
"id": 1,
"name": "Default Group"
},
"userIdName": {
"id": 1,
"name": "Randy Lewis"
},
"lastConnected": 1423810405000,
"createdByIdName": {
"id": 1,
"name": "Randy Lewis"
},
"locked": false,
"traceOn": false,
"userEmail": "mdm.user1#gmail.com",
"enrollmentDate": 1423790255000,
"simNumbers": [
"1236547896"
],
"imeiCodes": [],
"mobileNumbers": [
"4558585858858"
],
"createDate": 1421927250000,
"lastModified": 1423810406000,
"rooted": false,
"roaming": false,
"compliant": false
},
{
"apiResponseStatus": "SUCCESS",
"id": 24,
"ownership": "CORPORATE",
"deviceName": "Photobean",
"deviceType": "MOBILE",
"osType": "ANDROID",
"inTotalStorage": 20941258752,
"inAvailableStorage": 8024924160,
"exTotalStorage": 0,
"exAvailableStorage": 0,
"bluetoothMacAddress": "8C:3A:E3:BE:C2:04",
"wirelessMacAddress": "8c:3a:e3:51:bb:e3",
"model": "LG-D686",
"osVersion": "4.4.2",
"buildVersion": "1.01.09",
"status": "PENDING",
"isDeleted": false,
"policyIdNames": {
"id": 3,
"name": "Test Policy 3"
},
"deviceGroupIdNames": {
"id": 1,
"name": "Default Group"
},
"userIdName": {
"id": 1,
"name": "Randy Lewis"
},
"make": "LGE",
"lastConnected": 1423648831000,
"createdByIdName": {
"id": 1,
"name": "Randy Lewis"
},
"locked": false,
"traceOn": false,
"userEmail": "mdm.user1#gmail.com",
"enrollmentDate": 1423628675000,
"simNumbers": [
"404909008623970"
],
"imeiCodes": [
"359004053152889",
"359004053152871"
],
"mobileNumbers": [
"638847339387"
],
"createDate": 1422435029000,
"lastModified": 1423648831000,
"rooted": true,
"roaming": false,
"compliant": false
},
{
"apiResponseStatus": "SUCCESS",
"id": 25,
"ownership": "CORPORATE",
"deviceName": "Livetube",
"deviceType": "MOBILE",
"osType": "ANDROID",
"inTotalStorage": 0,
"inAvailableStorage": 0,
"exTotalStorage": 0,
"exAvailableStorage": 0,
"osVersion": "4.2",
"buildVersion": "1.01.08",
"status": "APPROVAL_PENDING",
"isDeleted": false,
"policyIdNames": {
"id": 3,
"name": "Test Policy 3"
},
"deviceGroupIdNames": {
"id": 1,
"name": "Default Group"
},
"userIdName": {
"id": 1,
"name": "Randy Lewis"
},
"lastConnected": 1423810522000,
"createdByIdName": {
"id": 1,
"name": "Randy Lewis"
},
"locked": false,
"traceOn": false,
"userEmail": "mdm.user1#gmail.com",
"enrollmentDate": 1423790371000,
"simNumbers": [
"1236547896"
],
"imeiCodes": [],
"mobileNumbers": [
"4558585858858"
],
"createDate": 1422435030000,
"lastModified": 1423810522000,
"rooted": false,
"roaming": false,
"compliant": false
},
{
"apiResponseStatus": "SUCCESS",
"id": 26,
"ownership": "CORPORATE",
"deviceName": "Riffwire",
"deviceType": "MOBILE",
"osType": "IOS",
"inTotalStorage": 0,
"inAvailableStorage": 0,
"exTotalStorage": 0,
"exAvailableStorage": 0,
"status": "PENDING",
"isDeleted": false,
"policyIdNames": {
"id": 3,
"name": "Test Policy 3"
},
"deviceGroupIdNames": {
"id": 1,
"name": "Default Group"
},
"userIdName": {
"id": 1,
"name": "Randy Lewis"
},
"createdByIdName": {
"id": 1,
"name": "Randy Lewis"
},
"locked": false,
"traceOn": false,
"userEmail": "mdm.user1#gmail.com",
"simNumbers": [],
"imeiCodes": [],
"mobileNumbers": [],
"createDate": 1422435030000,
"lastModified": 1423216312000,
"rooted": false,
"roaming": false,
"compliant": false
},
Also, Please suggest any good option if any.
You question is JMeter specific, but i believe it's actually regular Json Path expression. For example, in SoapUI it could be done with
$.deviceResponseList[?(#.status=='PENDING')].id
Probably this could help in some way
JMeter provides JSON Path Extractor via JMeter Plugins Extras With Libs Set
I believe that this is the best way to accomplish your goal.
See Using the XPath Extractor in JMeter guide (scroll down to "Parsing JSON" chapter) for the plugin installation instructions and some sample JSON Path queries.