diff --git a/all.csv b/all.csv new file mode 100644 index 0000000..1a3ceba --- /dev/null +++ b/all.csv @@ -0,0 +1,291 @@ +js-client-sdk,appMetadata,2.22 +js-client-sdk,bigSegments,1.0 +js-client-sdk,bootstrapping,2.5 +js-client-sdk,contexts,3.0 +js-client-sdk,experimentation,2.6 +js-client-sdk,variationDetail,2.72 +js-client-sdk,allFlags,1.0 +js-client-sdk,privateAttrs,2.9.4 +js-client-sdk,relayProxyProxy,1.0 +js-client-sdk,secureMode,2.17 +js-client-sdk,track,1.6.2 +js-client-sdk,flagChanges,1.0 +node-client-sdk,appMetadata,3.0 +node-client-sdk,bigSegments,1.0 +node-client-sdk,bootstrapping,1.0 +node-client-sdk,contexts,3.0 +node-client-sdk,experimentation,1.0 +node-client-sdk,flagChanges,2.0 +node-client-sdk,allFlags,1.0 +node-client-sdk,privateAttrs,1.0 +node-client-sdk,relayProxyProxy,1.0 +node-client-sdk,secureMode,1.4 +node-client-sdk,track,1.0 +node-client-sdk,flagChanges,1.0 +react-native,appMetadata,7.0 +react-native,bigSegments,1.0 +react-native,experimentation,5.0 +react-native,flagChanges,4.1.2 +react-native,allFlags,5.0.1 +react-native,multiEnv,4.2.0 +react-native,offlineMode,1.0 +react-native,privateAttrs,4.0 +react-native,relayProxyProxy,1.0 +react-native,track,1.0 +react-native,flagChanges,1.0 +react-client-sdk,appMetadata,2.26 +react-client-sdk,bigSegments,1.0 +react-client-sdk,bootstrapping,2.9.5 +react-client-sdk,contexts,3.0 +react-client-sdk,experimentation,1.0 +react-client-sdk,flagChanges,2.21 +react-client-sdk,allFlags,2.9 +react-client-sdk,relayProxyProxy,2.20.2 +dotnet-client-sdk,appMetadata,3.1 +dotnet-client-sdk,autoEnvAttrs,4.0 +dotnet-client-sdk,contexts,3.0 +dotnet-client-sdk,experimentation,2.0 +dotnet-client-sdk,variationDetail,1.2 +dotnet-client-sdk,allFlags,1.0 +dotnet-client-sdk,offlineMode,1.0 +dotnet-client-sdk,privateAttrs,1.0 +dotnet-client-sdk,relayProxyProxy,1.0 +dotnet-client-sdk,flagChanges,2.0 +dotnet-client-sdk,testDataSource,1.0 +dotnet-client-sdk,webProxy,1.0 +lua-server-sdk,experimentation,1.0 +lua-server-sdk,variationDetail,1.0 +lua-server-sdk,allFlags,1.0 +lua-server-sdk,track,1.0 +lua-server-sdk,storingData,1.0 +lua-server-sdk,relayProxyDaemon,1.0 +lua-server-sdk,relayProxyProxy,1.0 +cpp-server-sdk,appMetadata,3.0 +cpp-server-sdk,bigSegments,1.0 +cpp-server-sdk,contexts,3.0 +cpp-server-sdk,experimentation,2.4.8 +cpp-server-sdk,variationDetail,1.0 +cpp-server-sdk,allFlags,1.0 +cpp-server-sdk,offlineMode,1.0 +cpp-server-sdk,privateAttrs,1.0 +cpp-server-sdk,fileDataSource,2.6 +cpp-server-sdk,relayProxyProxy,1.0 +cpp-server-sdk,flagChanges,1.0 +cpp-server-sdk,testDataSource,2.6 +cpp-server-sdk,webProxy,1.0 +cpp-client-sdk,appMetadata,3.0 +cpp-client-sdk,bigSegments,1.0 +cpp-client-sdk,contexts,3.0 +cpp-client-sdk,experimentation,2.4.8 +cpp-client-sdk,variationDetail,1.0 +cpp-client-sdk,allFlags,1.0 +cpp-client-sdk,privateAttrs,1.0 +cpp-client-sdk,relayProxyProxy,1.0 +cpp-client-sdk,flagChanges,1.0 +cpp-client-sdk,testDataSource,1.0 +flutter-client-sdk,appMetadata,1.3 +flutter-client-sdk,bigSegments,1.0 +flutter-client-sdk,contexts,2.0 +flutter-client-sdk,experimentation,0.2.0 +flutter-client-sdk,variationDetail,1.0 +flutter-client-sdk,allFlags,1.0 +flutter-client-sdk,privateAttrs,1.0 +flutter-client-sdk,relayProxyProxy,1.0 +flutter-client-sdk,flagChanges,1.0 +flutter-client-sdk,track,1.0 +swift-client-sdk,appMetadata,8.0 +swift-client-sdk,bigSegments,1.0 +swift-client-sdk,contexts,8.0 +swift-client-sdk,experimentation,1.0 +swift-client-sdk,variationDetail,4.3 +swift-client-sdk,allFlags,1.0 +swift-client-sdk,privateAttrs,2.10 +swift-client-sdk,relayProxyProxy,1.0 +swift-client-sdk,flagChanges,1.0 +swift-client-sdk,secureMode,2.0 +swift-client-sdk,track,1.0 +roku,appMetadata,1.3 +roku,bigSegments,1.0 +roku,contexts,2.0 +roku,experimentation,1.0 +roku,variationDetail,1.0 +roku,allFlags,1.0 +roku,privateAttrs,1.0 +roku,relayProxyProxy,1.0 +node-server,appMetadata,6.4 +node-server,contexts,7.0 +node-server,experimentation,6.1 +node-server,variationDetail,5.4 +node-server,allFlags,3.0 +node-server,multiEnv,9.0 +node-server,offlineMode,1.0 +node-server,privateAttrs,3.3 +node-server,relayProxyDaemon,1.0 +node-server,relayProxyProxy,1.0 +node-server,secureMode,3.0 +node-server,track,1.0 +node-server,storingData,2.0 (Redis) +node-server,flagChanges,1.0 +node-server,webProxy,1.0 +php-server-sdk,appMetadata,4.3 +php-server-sdk,contexts,5.0 +php-server-sdk,experimentation,4.1 +php-server-sdk,variationDetail,3.4 +php-server-sdk,allFlags,1.6 +php-server-sdk,offlineMode,1.0 +php-server-sdk,privateAttrs,2.4 +php-server-sdk,relayProxyDaemon,1.0 +php-server-sdk,relayProxyProxy,1.0 +php-server-sdk,secureMode,1.0 +php-server-sdk,track,1.0 +php-server-sdk,storingData,1.0 (Redis), 3.5 (Consul and DynamoDB) +php-server-sdk,testDataSource,4.2 +python-server-sdk,appMetadata,7.6 +python-server-sdk,contexts,8.0 +python-server-sdk,experimentation,7.2 +python-server-sdk,variationDetail,6.4 +python-server-sdk,allFlags,2.0 +python-server-sdk,offlineMode,2.0 +python-server-sdk,privateAttrs,4.1 +python-server-sdk,relayProxyDaemon,2.0 +python-server-sdk,relayProxyProxy,2.0 +python-server-sdk,secureMode,2.0 +python-server-sdk,track,2.0 +python-server-sdk,storingData,2.0 (Consul and Redis), 6.7 (DynamoDB) +python-server-sdk,flagChanges,9.1 +python-server-sdk,testDataSource,7.4 +python-server-sdk,webProxy,2.0 +ruby-server-sdk,appMetadata,6.4 +ruby-server-sdk,contexts,7.0 +ruby-server-sdk,experimentation,6.2 +ruby-server-sdk,variationDetail,5.2 +ruby-server-sdk,allFlags,2.0.1 +ruby-server-sdk,offlineMode,2.0 +ruby-server-sdk,privateAttrs,2.0 +ruby-server-sdk,relayProxyDaemon,2.0 +ruby-server-sdk,relayProxyProxy,2.0 +ruby-server-sdk,secureMode,2.0.1 +ruby-server-sdk,track,2.0 +ruby-server-sdk,storingData,2.3.1 (Redis), 5.5 (Consul and DynamoDB) +ruby-server-sdk,flagChanges,2.0 +ruby-server-sdk,testDataSource,2.0 +ruby-server-sdk,webProxy,2.2.5 +rust-server-sdk,appMetadata,1.0 +rust-server-sdk,contexts,1.0 +rust-server-sdk,experimentation,1.0 +rust-server-sdk,variationDetail,1.0 +rust-server-sdk,allFlags,1.0 +rust-server-sdk,offlineMode,1.0 +rust-server-sdk,privateAttrs,1.0 +rust-server-sdk,relayProxyProxy,1.0 +rust-server-sdk,secureMode,1.0 +rust-server-sdk,track,1.0 +cloudflare,contexts,2.0 +cloudflare,experimentation,2.3.0 +cloudflare,allFlags,1.0 +cloudflare,secureMode,2.0 +cloudflare,track,2.3 +vercel,contexts,1.0 +vercel,experimentation,1.2.0 +vercel,allFlags,1.0 +vercel,secureMode,1.0 +vercel,track,1.2 +vue,bigSegments,1.0 +vue,contexts,2.0 +vue,experimentation,1.0 +vue,variationDetail,1.0 +vue,privateAttrs,1.0 +akamai,contexts,1.0 +android,privateAttrs,2.2 +android,relayProxyProxy,1.0 +android,flagChanges,1.0 +apex-server-sdk,experimentation,1.1 +apex-server-sdk,variationDetail,1.0 +apex-server-sdk,allFlags,1.0 +apex-server-sdk,privateAttrs,1.0 +apex-server-sdk,track,1.0 +apex-server-sdk,webProxy,1.0 +dotnet-server-sdk,appMetadata,8.0 +dotnet-server-sdk,bigSegments,6.2 +dotnet-server-sdk,contexts,7.0 +dotnet-server-sdk,experimentation,6.1 +dotnet-server-sdk,variationDetail,5.4 +dotnet-server-sdk,allFlags,2.0 +dotnet-server-sdk,migrations,8.0 +dotnet-server-sdk,offlineMode,1.0 +dotnet-server-sdk,otel,8.4 +dotnet-server-sdk,privateAttrs,3.5 +dotnet-server-sdk,fileDataSource,5.5 +dotnet-server-sdk,relayProxyDaemon,5.2 +dotnet-server-sdk,relayProxyProxy,1.0 +dotnet-server-sdk,secureMode,2.0 +dotnet-server-sdk,track,1.0 +dotnet-server-sdk,storingData,1.0 +dotnet-server-sdk,flagChanges,6.0 +dotnet-server-sdk,testDataSource,6.0 +dotnet-server-sdk,webProxy,1.0 +go-server-sdk,appMetadata,5.9 +go-server-sdk,bigSegments,5.5 +go-server-sdk,contexts,6.0 +go-server-sdk,experimentation,5.4 +go-server-sdk,variationDetail,4.3 +go-server-sdk,allFlags,2.0 +go-server-sdk,migrations,7.0 +go-server-sdk,offlineMode,2.0 +go-server-sdk,otel,7.4 +go-server-sdk,privateAttrs,2.2.3 +go-server-sdk,fileDataSource,4.4 +go-server-sdk,relayProxyDaemon,2.0 +go-server-sdk,relayProxyProxy,2.0 +go-server-sdk,secureMode,2.0 +go-server-sdk,track,2.0 +go-server-sdk,storingData,4.5 (DynamoDB, Consul) +go-server-sdk,flagChanges,5.0 +go-server-sdk,testDataSource,2.0 +go-server-sdk,webProxy,2.0 +java-server-sdk,bigSegments,5.7 +java-server-sdk,experimentation,5.5 +java-server-sdk,variationDetail,4.3 +java-server-sdk,allFlags,1.0 +java-server-sdk,migrations,7.0 +java-server-sdk,offlineMode,1.0 +java-server-sdk,otel,7.4 +java-server-sdk,privateAttrs,1.0 +java-server-sdk,fileDataSource,4.5 +java-server-sdk,relayProxyDaemon,1.0 +java-server-sdk,relayProxyProxy,1.0 +java-server-sdk,secureMode,1.0 +java-server-sdk,track,1.0 +java-server-sdk,storingData,1.0 +java-server-sdk,flagChanges,5.0 +java-server-sdk,testDataSource,5.1 +haskell-server-sdk,appMetadata,3.1 +haskell-server-sdk,contexts,4.0 +haskell-server-sdk,experimentation,2.2 +haskell-server-sdk,variationDetail,1.0 +haskell-server-sdk,allFlags,1.0 +haskell-server-sdk,offlineMode,1.0 +haskell-server-sdk,privateAttrs,2.5 +haskell-server-sdk,fileDataSource,3.0 +haskell-server-sdk,track,1.0 +haskell-server-sdk,testDataSource,1.0 +erlang-server-sdk,appMetadata,1.6 +erlang-server-sdk,contexts,2.0 +erlang-server-sdk,experimentation,1.2 +erlang-server-sdk,variationDetail,1.0 +erlang-server-sdk,allFlags,1.0 +erlang-server-sdk,offlineMode,1.0 +erlang-server-sdk,privateAttrs,1.0 +erlang-server-sdk,fileDataSource,1.0 +erlang-server-sdk,track,1.0 +erlang-server-sdk,testDataSource,1.4 +electron,bigSegments,1.0 +electron,bootstrapping,1.0 +electron,experimentation,1.0 +electron,variationDetail,1.5.7 +electron,allFlags,1.0 +electron,privateAttrs,1.0.1 +electron,relayProxyProxy,1.5.6 +electron,secureMode,1.5 +electron,track,1.1.4 +electron,flagChanges,1.0 diff --git a/raw_data.csv b/raw_data.csv new file mode 100644 index 0000000..96a8261 --- /dev/null +++ b/raw_data.csv @@ -0,0 +1,225 @@ + Here is some raw HTML. It consists of some headers (which are SDK_IDs) and rows (which are FEATURE_NAMEs, with + associated version values.) + + I want to extract all the data into a clean csv format. + +For example, under the [Electron] header, the values for the features are: +Bootstrapping = 1.0 +Experimentation = 1.0 +Flag evaluation reasons=1.5.7 + +and so on. + +The resulting part of hte CSV would look like + +sdk,feature,version +electron,bootstrapping,1.0 +electron,experimentation,1.0 +electron,flag evaluation reasons,1.5.7 + +Data begins here: + + + + Feature + [Electron](/sdk/client-side/electron) + [JavaScript](/sdk/client-side/javascript) + [Node.js](/sdk/client-side/node-js) + [React Native](/sdk/client-side/react/react-native) + [React Web](/sdk/client-side/react/react-web) + [Vue](/sdk/client-side/vue) + + + + [Application metadata](/sdk/features/app-config) + + 2.22 + 3.0 + 7.0 + 2.26 + + + + [Automatic environment attributes](/sdk/features/environment-attributes) + + + + 8.0 + + + + + [Big segments](/sdk/features/big-segments) + All + All + All + All + All + All + + + [Bootstrapping](/sdk/features/bootstrapping) + 1.0 + 2.5 + 1.0 + + 2.9.5 + + + + [Contexts and context kinds](/sdk/features/user-context-config) + + 3.0 + 3.0 + 7.0 + 3.0 + 2.0 + + + [Experimentation](/home/experimentation) + 1.0 + 2.6 + 1.0 + 5.0 + 1.0 + 1.0 + + + [Flag evaluation reasons](/sdk/features/evaluation-reasons) + 1.5.7 + 2.72 + 2.0 + 4.1.2 + 2.21 + 1.0 + + + [Getting all flags](/sdk/features/all-flags) + 1.0 + 1.0 + 1.0 + 5.0.1 + 2.9 + + + + [Migration flags](/sdk/features/migrations) + + + + + + + + + [Multiple environments](/sdk/features/multiple-environments) + + + + 4.2.0 + + + + + [Offline mode](/sdk/features/offline-mode) + + + + 1.0 + + + + + [Private attributes](/sdk/features/private-attributes) + 1.0.1 + 2.9.4 + 1.0 + 4.0 + 2.9 + 1.0 + + + [Reading flags from a file](/sdk/features/flags-from-files) + + + + + + + + + [Relay Proxy in daemon mode](/sdk/features/relay-proxy-configuration/daemon-mode) + + + + + + + + + [Relay Proxy in proxy mode](/sdk/features/relay-proxy-configuration/proxy-mode) + 1.5.6 + 1.0 + 1.0 + 1.0 + 2.20.2 + + + + [Secure mode](/sdk/features/secure-mode) + 1.5 + 2.17 + 1.4 + + 2.18 + + + + [Sending custom events](/sdk/features/events) + 1.1.4 + 1.6.2 + 1.0 + 1.0 + + + + + [Storing data](/sdk/features/storing-data) + + + + + + + + + [Subscribing to flag changes](/sdk/features/flag-changes) + 1.0 + 1.0 + 1.0 + 1.0 + 2.9 + + + + [Test data sources](/sdk/features/test-data-sources) + + + + + + + + + [Web proxy configuration](/sdk/features/web-proxy) + + + + + + + + +
+ +The CSV is as follows: diff --git a/scripts/add-all-features.sh b/scripts/add-all-features.sh new file mode 100755 index 0000000..debd0ee --- /dev/null +++ b/scripts/add-all-features.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -e + +csv=./all.csv + +create_prs=false +copy_metadata=false + +if [ "$1" == "-create-prs" ]; then + create_prs=true + shift +fi + +if [ "$1" == "-copy-metadata" ]; then + copy_metadata=true + shift +fi + + +ids=$(jq -r 'keys[]' ./products/names.json | grep -v relay-proxy) + +for id in $ids; do + repo_link=$(jq -r ".\"$id\".github" ./products/repos.json) + repo=$(echo "$repo_link" | cut -d'/' -f2) + grep "$id" "$csv" | while IFS=, read -r sdk_id feature version; do + echo "Adding feature $feature to $repo at version $version" + ./scripts/add-feature.sh "$repo/$id" "$feature" "$version" + done +done + +GH_USERNAME=$(gh api user | jq -r .login) + +if [ "$create_prs" == true ]; then + echo "Creating PRs" + for id in $ids; do + repo_link=$(jq -r ".\"$id\".github" ./products/repos.json) + # The linke consists of launchdarkly/sdk-id. Strip off the launchdarkly/ part to get + # the repo only: + repo=$(echo "$repo_link" | cut -d'/' -f2) + + ( + cd "$repo" || exit + git switch -c "$GH_USERNAME"/update-version-metadata + git add .sdk_metadata.json + git commit -m "chore: update .sdk_metadata.json with feature table" + gh pr create --fill + ) + + + done +fi + +if [ "$copy_metadata" == true ]; then + echo "Copying metadata" + for id in $ids; do + repo_link=$(jq -r ".\"$id\".github" ./products/repos.json) + + repo=$(echo "$repo_link" | cut -d'/' -f2) + + cp "$repo/.sdk_metadata.json" "./backfill/launchdarkly_$repo.json" + done +fi diff --git a/scripts/add-feature.sh b/scripts/add-feature.sh new file mode 100755 index 0000000..3cb926d --- /dev/null +++ b/scripts/add-feature.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +# This script can be used to add a feature (or deprecate one) for a particular SDK. +# For example, using a monorepo: ./add-feature.sh cpp-sdks/cpp-client-sdk appMetadata 3.0 +# Or a solorepo: ./add-feature.sh ruby-server-sdk appMetadata 3.0 +repo=$1 +feature=$2 +introduced=$3 + +echo "The repo: $repo" + +if [ -z "$repo" ] || [ -z "$feature" ] || [ -z "$introduced" ]; then + echo "Usage: $0 " + exit 1 +fi + + +sqlite3 metadata.sqlite3 "SELECT COUNT(*) FROM sdk_feature_info WHERE id = '$feature';" | + grep 1 || { + echo "Feature '$feature' not recognized. Check spelling, or add it to the metadata.sqlite3 database first."; + echo "Known features:" + sqlite3 metadata.sqlite3 "SELECT id FROM sdk_feature_info;" + exit 1; + } + +# The repo name is either going to be something like ruby-server-sdk, +# or a SDK within a monorepo, like js-core/node-server-sdk. +# If there's a slash, we need to split it into the two components so that we can clone the first one +# in both cases. + +if [[ "$repo" == *"/"* ]]; then + repo_name=$(echo "$repo" | cut -d'/' -f1) + is_monorepo=true +else + repo_name=$repo + is_monorepo=false +fi + +if [ -d "$repo_name" ]; then + echo "note: repo already cloned, will update" +else + gh repo clone "launchdarkly/$repo_name" -- --depth=1 +fi + +if [ ! -f "$repo_name/.sdk_metadata.json" ]; then + echo ".sdk_metadata.json not found, run ./scripts/add-repo.sh first" + exit 1 +fi + +# Now, if is_monorepo is true, then we need to access the only child SDK of the "sdks" key in the .sdk_metadata.json +# file. That is, the file will have an "sdks" : { "some-sdk-id" : { .. } } structure, and we need to automatically +# be editing the object some-sdk-id. +# Otherwise, if this is a monorepo, we need to edit the child object of "sdks" that matches the sdk name (the second +# part of the repo name after the slash.) +# What we'll do is add a new sub key, if it doesn't exist, named "features". Then we'll add the feature name +# that was specified in the argument. The value is an object with a single key, "introduced". +# Then we'll set "introduced"'s value to the version specified as an argument. If the feature +# key already exists, we'll update it's value to the new version and echo a warning. +# Here's how to do it in jq: + +sdk_id=$(basename "$repo") + +if [ "$is_monorepo" = true ]; then + jq --arg feature "$feature" --arg introduced "$introduced" --arg sdk "$sdk_id" '.sdks[$sdk].features |= . + { ($feature): { introduced: $introduced } }' "$repo_name/.sdk_metadata.json" > "$repo_name/.sdk_metadata.json.tmp" +else + jq --arg feature "$feature" --arg introduced "$introduced" '.sdks[.sdks | keys[0]].features |= . + { ($feature): { introduced: $introduced } }' "$repo_name/.sdk_metadata.json" > "$repo_name/.sdk_metadata.json.tmp" +fi + +mv "$repo_name/.sdk_metadata.json.tmp" "$repo_name/.sdk_metadata.json" diff --git a/scripts/repos.sh b/scripts/repos.sh index 40d6247..0bbd747 100755 --- a/scripts/repos.sh +++ b/scripts/repos.sh @@ -10,6 +10,7 @@ gh api --paginate graphql -f query='{ nodes { ... on Repository { nameWithOwner + isArchived repositoryTopics(first: 100) { nodes { topic {