From dd5fc9f83c290cc87c6e55e499b90d25e9453ee1 Mon Sep 17 00:00:00 2001
From: Derek Wang <derek.kiah.wang@gmail.com>
Date: Fri, 21 Feb 2025 00:39:25 -0800
Subject: [PATCH] Abstracted data loading so that data can be loaded simply
through the config file. Moved Mike's LLM data generation code and my own
dummy data generating code to the database folder.
---
app/config.py | 1 +
app/config.yaml | 1 +
app/database.py | 16 +-
app/database/create_basic_dummy_data.py | 44 +++++
app/database/datastore/basicDummy_sns.json | 214 +++++++++++++++++++++
app/database/datastore/llmData_sns.json | 1 +
app/{ => database}/generate_data.py | 16 +-
7 files changed, 284 insertions(+), 9 deletions(-)
create mode 100644 app/database/create_basic_dummy_data.py
create mode 100644 app/database/datastore/basicDummy_sns.json
create mode 100644 app/database/datastore/llmData_sns.json
rename app/{ => database}/generate_data.py (87%)
diff --git a/app/config.py b/app/config.py
index fcdc507..c4b3b9a 100644
--- a/app/config.py
+++ b/app/config.py
@@ -13,3 +13,4 @@ config = load_config()
CACHE_STRATEGY = os.getenv("CACHE_STRATEGY", config.get("cache_strategy", "Baseline"))
CACHE_LIMIT = config.get("cache_limit", 10)
L2_CACHE_LIMIT = config.get("l2_cache_limit", 100)
+DB_FILE = config.get("db_file", "llmData_sns.json")
diff --git a/app/config.yaml b/app/config.yaml
index add1fa1..c8eb188 100644
--- a/app/config.yaml
+++ b/app/config.yaml
@@ -1,3 +1,4 @@
cache_strategy: "Baseline" # Change this to "Prefetch" or "Tiered" or "Seive"
cache_limit: 10
l2_cache_limit: 100
+db_file: "llmData_sns.json" # Change this to the name of any json file within the "database/datastore" folder
\ No newline at end of file
diff --git a/app/database.py b/app/database.py
index fb3f17f..9baf5ab 100644
--- a/app/database.py
+++ b/app/database.py
@@ -1,10 +1,11 @@
from tinydb import TinyDB, Query
-from generate_data import generate_data
+from config import DB_FILE
+
+DB_LOCATION = "database/datastore/" + DB_FILE
# Initialize TinyDB as a NoSQL key-value store
-DB_FILE = "database.json"
-db = TinyDB(DB_FILE)
+db = TinyDB(DB_LOCATION)
User = Query()
def get_user_profile(user_id):
@@ -19,9 +20,12 @@ def update_user_profile(user_id, name, followers, bio, posts, friends):
def init_db():
"""Ensure TinyDB is initialized before FastAPI starts and prepopulate some data"""
global db
- db = TinyDB(DB_FILE) # Reload TinyDB if needed
+ db = TinyDB(DB_LOCATION) # Reload TinyDB if needed
# Prepopulate database with some sample users if empty
if len(db) == 0:
- data = generate_data(100)
- db.insert_multiple(data)
+ db.insert_multiple([
+ {"user_id": "1", "name": "Alice", "followers": 100, "bio": "Love coding!", "posts": "Hello, world!"},
+ {"user_id": "2", "name": "Bob", "followers": 200, "bio": "Tech enthusiast", "posts": "AI is amazing!"},
+ {"user_id": "3", "name": "Charlie", "followers": 50, "bio": "Blogger", "posts": "Check out my latest post!"}
+ ])
diff --git a/app/database/create_basic_dummy_data.py b/app/database/create_basic_dummy_data.py
new file mode 100644
index 0000000..7f045e4
--- /dev/null
+++ b/app/database/create_basic_dummy_data.py
@@ -0,0 +1,44 @@
+# Quick script to create basic dummy data for the caching system
+# Current theme/usage is in a social media microservice setting such as Twitter
+# Current database schema:
+### user_id (primary key), str
+### username, string
+### num_followers, int
+### posts, str (for now, can be list later on)
+### friends, list[user_id]
+
+import json
+import random
+
+# Parameters to change the distribution/random ranges
+TOTAL_USERS = 20
+MIN_FOLLOWERS, MAX_FOLLOWERS = 0, 5
+MIN_POSTS, MAX_POSTS = 5, 10
+MIN_FRIENDS, MAX_FRIENDS = 1, 5
+
+# Create the user data
+# TODO if we want to vary the user data more, we can inflate the posts into a list of strings and make the strings very long :3"
+data = {}
+for user_id in range( TOTAL_USERS ):
+ user = {}
+ user["user_id"] = str( user_id )
+ user["username"] = "user" + str( user_id )
+ user["num_followers"] = random.randint( MIN_FOLLOWERS, MAX_FOLLOWERS )
+
+ # Just make a single post for now, can consider mutliple posts later
+ user["posts"] = f"This is user {user_id}'s post!"
+
+ # posts = []
+ # for post_num in range( random.randint( MIN_FOLLOWERS, MAX_FOLLOWERS ) ):
+ # posts.append( f"This is user {user_id}'s post {str( post_num + 1 )}!" )
+ # user["posts"] = posts
+
+ friends = []
+ user["friends"] = random.sample( [u_id for u_id in range( TOTAL_USERS ) if u_id != user_id], random.randint( MIN_FOLLOWERS, MAX_FOLLOWERS ) )
+
+ data[user_id] = user
+
+# Load the data into a json object and write it into the datastore folder
+json_object = json.dumps( {"_default": data}, indent=4 )
+with open( "datastore/basicDummy_sns.json", "w" ) as f:
+ f.write( json_object )
diff --git a/app/database/datastore/basicDummy_sns.json b/app/database/datastore/basicDummy_sns.json
new file mode 100644
index 0000000..ba5fcbd
--- /dev/null
+++ b/app/database/datastore/basicDummy_sns.json
@@ -0,0 +1,214 @@
+{
+ "_default": {
+ "0": {
+ "user_id": "0",
+ "username": "user0",
+ "num_followers": 0,
+ "posts": "This is user 0's post!",
+ "friends": [
+ 17,
+ 13,
+ 14
+ ]
+ },
+ "1": {
+ "user_id": "1",
+ "username": "user1",
+ "num_followers": 1,
+ "posts": "This is user 1's post!",
+ "friends": [
+ 8,
+ 17,
+ 15,
+ 3
+ ]
+ },
+ "2": {
+ "user_id": "2",
+ "username": "user2",
+ "num_followers": 4,
+ "posts": "This is user 2's post!",
+ "friends": [
+ 9,
+ 12
+ ]
+ },
+ "3": {
+ "user_id": "3",
+ "username": "user3",
+ "num_followers": 1,
+ "posts": "This is user 3's post!",
+ "friends": [
+ 10,
+ 7,
+ 8,
+ 14
+ ]
+ },
+ "4": {
+ "user_id": "4",
+ "username": "user4",
+ "num_followers": 0,
+ "posts": "This is user 4's post!",
+ "friends": [
+ 13,
+ 0
+ ]
+ },
+ "5": {
+ "user_id": "5",
+ "username": "user5",
+ "num_followers": 4,
+ "posts": "This is user 5's post!",
+ "friends": []
+ },
+ "6": {
+ "user_id": "6",
+ "username": "user6",
+ "num_followers": 3,
+ "posts": "This is user 6's post!",
+ "friends": []
+ },
+ "7": {
+ "user_id": "7",
+ "username": "user7",
+ "num_followers": 4,
+ "posts": "This is user 7's post!",
+ "friends": [
+ 15,
+ 13,
+ 11,
+ 17
+ ]
+ },
+ "8": {
+ "user_id": "8",
+ "username": "user8",
+ "num_followers": 2,
+ "posts": "This is user 8's post!",
+ "friends": [
+ 4,
+ 19
+ ]
+ },
+ "9": {
+ "user_id": "9",
+ "username": "user9",
+ "num_followers": 0,
+ "posts": "This is user 9's post!",
+ "friends": [
+ 2,
+ 10
+ ]
+ },
+ "10": {
+ "user_id": "10",
+ "username": "user10",
+ "num_followers": 5,
+ "posts": "This is user 10's post!",
+ "friends": [
+ 7,
+ 12
+ ]
+ },
+ "11": {
+ "user_id": "11",
+ "username": "user11",
+ "num_followers": 1,
+ "posts": "This is user 11's post!",
+ "friends": [
+ 12,
+ 8,
+ 18,
+ 4
+ ]
+ },
+ "12": {
+ "user_id": "12",
+ "username": "user12",
+ "num_followers": 1,
+ "posts": "This is user 12's post!",
+ "friends": [
+ 5,
+ 17,
+ 8,
+ 4
+ ]
+ },
+ "13": {
+ "user_id": "13",
+ "username": "user13",
+ "num_followers": 4,
+ "posts": "This is user 13's post!",
+ "friends": [
+ 16,
+ 10,
+ 2,
+ 8,
+ 1
+ ]
+ },
+ "14": {
+ "user_id": "14",
+ "username": "user14",
+ "num_followers": 3,
+ "posts": "This is user 14's post!",
+ "friends": [
+ 11
+ ]
+ },
+ "15": {
+ "user_id": "15",
+ "username": "user15",
+ "num_followers": 2,
+ "posts": "This is user 15's post!",
+ "friends": [
+ 13,
+ 14
+ ]
+ },
+ "16": {
+ "user_id": "16",
+ "username": "user16",
+ "num_followers": 5,
+ "posts": "This is user 16's post!",
+ "friends": [
+ 13,
+ 3,
+ 14
+ ]
+ },
+ "17": {
+ "user_id": "17",
+ "username": "user17",
+ "num_followers": 3,
+ "posts": "This is user 17's post!",
+ "friends": [
+ 0,
+ 19,
+ 2,
+ 16
+ ]
+ },
+ "18": {
+ "user_id": "18",
+ "username": "user18",
+ "num_followers": 4,
+ "posts": "This is user 18's post!",
+ "friends": [
+ 9
+ ]
+ },
+ "19": {
+ "user_id": "19",
+ "username": "user19",
+ "num_followers": 1,
+ "posts": "This is user 19's post!",
+ "friends": [
+ 1,
+ 12,
+ 9
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/app/database/datastore/llmData_sns.json b/app/database/datastore/llmData_sns.json
new file mode 100644
index 0000000..b0a67b6
--- /dev/null
+++ b/app/database/datastore/llmData_sns.json
@@ -0,0 +1 @@
+{"_default": {"1": {"user_id": "0", "name": "Alex Taylor", "followers": 1295, "bio": "Avid reader and bookworm", "posts": "Just finished", "friends": ["2", "89", "20", "66", "13", "46", "58", "76", "75", "53", "95", "55", "23", "61", "71", "33", "87", "80", "86", "72", "83", "15", "30", "3", "47", "79", "6", "52", "74", "12", "26", "37", "69", "41", "45", "19", "85", "67", "9", "16", "63", "35", "24", "17", "51", "36", "38", "34", "7", "57", "81", "25", "93"]}, "2": {"user_id": "1", "name": "Alex Johnson", "followers": 3878, "bio": "Outdoor Adventures", "posts": "Completed a", "friends": ["15", "18", "11", "98", "79", "85", "57", "47", "87", "40", "32", "75"]}, "3": {"user_id": "2", "name": "John Doe", "followers": 122, "bio": "Tech enthusiast", "posts": "Just finished building a DIY smart home setup.", "friends": ["49", "45", "24", "86", "63"]}, "4": {"user_id": "11", "name": "Alex Taylor", "followers": 133, "bio": "Rock climbing and wildlife photography", "posts": "Uploaded a breathtaking photo of an eagle.", "friends": ["62", "40", "66", "36", "85", "59", "52", "20", "38", "50", "26", "24", "45", "46", "71", "56", "68", "12", "17", "39", "22", "10", "77", "79", "83", "94", "34", "1", "54", "43", "23", "74", "18", "65", "33", "16", "8", "84", "93", "41", "72", "5", "60", "70", "95", "73", "57", "75", "63", "25", "7", "44", "13", "78", "9", "47", "76", "48", "6", "35", "99", "69", "32", "67", "96", "89", "42", "19", "91", "3", "90", "37", "55", "58", "64", "0", "21", "27", "87", "82", "28", "98", "80", "53", "14", "49", "4"]}, "5": {"user_id": "14", "name": "Max Johnson", "followers": 1398, "bio": "Hiking & Camping", "posts": "Just completed a solo backpacking trip.", "friends": ["99", "67", "16", "45", "57", "34", "32", "39", "52", "64", "41", "6", "58", "79", "12", "94", "29", "20", "80", "47", "18", "26", "25", "48", "37", "3", "69", "4", "54", "31", "9", "87", "33", "49", "60", "84", "10", "23", "62", "56", "76", "96", "30", "68", "63", "15", "40", "44", "93", "46", "85", "81", "35", "72", "98", "65", "92", "61", "66", "11", "38", "28", "77", "97", "78", "42", "13", "27", "86", "55", "91", "19", "88", "8", "83", "73", "24", "95", "90", "53", "17", "74", "22", "36", "21", "43", "59", "71", "70", "82", "51", "50", "5", "89", "7"]}, "6": {"user_id": "15", "name": "Alex Morgan", "followers": 2538, "bio": "Loving life through sports and fitness!", "posts": "Just set a new personal best in the", "friends": ["25", "46", "71", "77", "66", "23", "72", "53", "19", "87", "32", "59", "41", "56", "33", "39", "93", "94", "92", "85", "88", "90", "37", "75", "55", "10", "24", "81", "80", "44", "47", "61", "76", "0", "6", "74", "49", "98", "4", "51", "42", "13", "3", "5", "60", "65", "91", "22", "27", "8", "14", "62", "43", "78", "69", "89"]}, "7": {"user_id": "16", "name": "Alex Taylor", "followers": 2595, "bio": "Music Enthusiast", "posts": "Just discovered a new favorite indie band.", "friends": ["24", "45", "36", "20", "82", "46", "13", "77", "54", "55", "11", "37", "27", "30", "26", "57", "73", "60", "85", "80", "75", "81", "34", "56", "25", "88", "49", "10", "42", "18", "90", "95", "64", "72", "69", "31", "52", "99", "22", "40", "29"]}, "8": {"user_id": "18", "name": "Alex Johnson", "followers": 2164, "bio": "Avid reader and traveler", "posts": "Just finished reading", "friends": ["82", "85", "11", "38", "5", "32", "65", "48", "8", "97", "54", "28", "14", "2", "60", "58", "50", "22", "90", "4", "70", "87", "21", "92", "74", "19", "84", "47", "63", "39", "57", "80", "96", "78"]}, "9": {"user_id": "19", "name": "Jordan Baker", "followers": 4851, "bio": "Art and Literature", "posts": "Finished reading 'The Catcher in the Rye'.", "friends": ["37", "23", "33", "92", "40", "14", "82", "75", "39", "76", "34", "7", "2", "93", "55", "4", "89", "74", "99", "80", "38", "58", "27", "29", "87", "77", "66", "32", "64", "17", "11", "20", "31", "78", "81", "97", "47", "94", "98", "35", "83", "42", "65", "45", "18", "79", "61", "46", "53", "28", "63", "90"]}, "10": {"user_id": "20", "name": "Bob Johnson", "followers": 393, "bio": "Music enthusiast & aspiring DJ", "posts": "Added new tracks to my 'Beats to Chill' playlist.", "friends": ["62", "81", "39", "88", "95", "28", "74", "45", "24", "43", "41", "76", "80", "10", "21", "71", "37", "2", "1", "11", "18", "9", "67", "0", "22", "25", "15", "57", "49", "56", "48", "50", "58", "75", "16", "86", "35", "46", "34", "51", "27", "92", "79", "68", "13", "72", "63", "59", "36", "23", "38", "32", "99", "96", "87", "3", "33", "6", "94", "60", "70", "97"]}, "11": {"user_id": "21", "name": "", "followers": 4490, "bio": "the user", "posts": "", "friends": ["27", "14", "29", "38", "7", "11", "54", "77", "92", "5", "81", "39", "25", "49", "94", "70", "41", "3", "13", "74", "75", "80", "20", "55", "85", "1", "31", "56", "53", "62", "2", "66", "22", "72", "37", "44", "86", "4", "68", "64", "67", "47", "24", "59", "83", "8", "73", "98", "43", "36", "35", "87", "10", "16", "42", "26", "40", "69", "0", "9", "91", "52", "33", "76", "18", "95", "50", "63", "97", "45", "12", "34", "15", "19", "96", "90", "46", "79", "88", "30", "99", "6", "84", "17", "51", "58", "93", "82", "78", "61"]}, "12": {"user_id": "22", "name": "Bob Johnson", "followers": 1934, "bio": "Avid reader and hiker", "posts": "Finished 'The Catcher in the Rye' and planning a hike this weekend.", "friends": ["93", "80", "53", "45", "72", "43", "20", "52", "46", "78", "12", "34", "44", "99", "47", "32", "88", "87", "31", "25", "15", "73", "16", "76", "85", "70", "92", "81", "61", "1", "66", "2"]}, "13": {"user_id": "24", "name": "Jean Valjean", "followers": 456, "bio": "Pursuit of knowledge and social justice", "posts": "Just finished reading 'Les Mis", "friends": ["69", "26", "80", "31", "68", "82", "14", "55", "45", "47", "32", "86", "35", "18", "49", "92", "44", "0", "57", "2", "46", "42", "60", "39", "20", "29", "7", "93", "25", "40", "81", "11", "94", "16", "72", "95", "66", "79", "58", "15", "30", "54", "70", "33", "65", "13", "97", "64", "41", "21", "74", "98", "53", "85", "27", "87", "9", "67", "8", "89", "1", "19", "84", "52", "91", "77", "12", "73", "28", "90", "51", "17", "6", "10", "50", "56"]}, "14": {"user_id": "29", "name": "Bob Johnson", "followers": 2342, "bio": "Hiking and photography", "posts": "Uploaded a new landscape picture.", "friends": ["43", "7", "85", "94", "63", "75", "52", "64", "51", "71", "11", "26", "57", "54", "39", "34", "93", "86", "84", "47", "58", "21", "31", "60", "48", "2", "68", "99", "96", "25", "59", "28", "8", "10", "88", "32", "35", "76", "55", "42"]}, "15": {"user_id": "30", "name": "Bob Johnson", "followers": 4205, "bio": "Tech Enthusiast", "posts": "Just posted an insightful blog about the future of AI.", "friends": ["95", "20", "73", "28", "29"]}, "16": {"user_id": "34", "name": "Alex Taylor", "followers": 348, "bio": "Avid Reader & Nature Lover", "posts": "Finished reading 'The Alchemist' and planning a weekend hike.", "friends": ["28", "41", "88", "99", "23", "66", "58", "69", "96", "17", "39", "84", "70", "32", "43", "52", "44", "98", "14", "29", "94", "19", "45", "72", "30", "55", "7", "60", "5", "57", "67", "76", "35", "63", "20", "38", "80", "16", "22", "49", "95", "18", "0", "93", "47", "81", "26", "59", "42", "24", "56", "15", "74", "78", "4", "25", "27"]}, "17": {"user_id": "35", "name": "Jack Black", "followers": 3355, "bio": "Rock music & cooking", "posts": "Added a new recipe to my food blog.", "friends": ["25", "22", "77", "72", "71", "55", "30", "27", "74", "49", "84", "20", "1", "23", "54", "78", "92", "36", "52", "32", "60", "37", "6", "69", "43", "18", "24", "67", "5", "48", "10", "0", "99", "85", "44", "46", "62", "28", "29", "50", "64", "81", "58", "45", "38", "88", "70", "61", "86", "42", "11", "66", "9", "80", "56", "34", "16"]}, "18": {"user_id": "37", "name": "Jordan Baker", "followers": 837, "bio": "Reading and hiking", "posts": "Finished", "friends": ["45", "74", "87", "34", "8", "35", "13", "23", "50", "58", "57", "71", "26", "6", "76", "83", "88", "67", "55", "54", "51", "97", "75", "47", "24", "4", "39", "15", "53", "70", "49"]}, "19": {"user_id": "38", "name": "Sam Green", "followers": 3983, "bio": "Hiking and photography enthusiast", "posts": "Recently snapped a great shot of a waterfall.", "friends": ["79", "30", "14", "45", "70", "85", "46", "95", "19", "53", "3", "7", "67", "82", "96", "57", "18", "75", "4", "86", "74", "26", "13", "47", "34", "50", "31", "15", "40", "54", "59", "51", "42", "39", "92", "80", "73", "2"]}, "20": {"user_id": "39", "name": "John Doe", "followers": 429, "bio": "Photography and nature", "posts": "Posted a picture of a lone wolf this morning.", "friends": ["26", "16", "12", "87", "31", "68", "93", "3", "54", "77", "36", "67", "56", "7", "14", "51", "83", "6", "40", "64", "91", "22"]}, "21": {"user_id": "40", "name": "Jamie Lee", "followers": 3995, "bio": "Avid reader and book collector", "posts": "Just finished", "friends": ["38", "1", "49", "77", "41", "75", "9", "48", "92", "71", "78", "88", "64", "11", "58", "95", "32"]}, "22": {"user_id": "41", "name": "Robert Johnson", "followers": 210, "bio": "Photography", "posts": "Just shared an intriguing street shot.", "friends": ["13", "30", "4", "31", "37", "94", "29", "80", "11", "22", "39", "57", "60", "12", "49", "34", "2", "78", "67", "70", "59", "6", "7", "99", "77", "87", "25", "79", "18", "44", "43", "33", "28", "89", "3", "52", "32", "36", "61", "62", "82", "46", "8", "48", "71", "65", "1", "55", "0", "10", "26", "66", "51", "85", "81", "91", "58", "16", "24", "86", "73", "97", "47", "63", "42", "72", "5", "90", "75", "53", "83", "14", "35", "17"]}, "23": {"user_id": "43", "name": "John Doe", "followers": 4120, "bio": "Avid reader & aspiring chef", "posts": "Just started a new book & trying out a new recipe.", "friends": ["51", "50", "16", "62", "30", "65", "60", "56", "91", "54", "69", "93", "32", "13", "46", "95", "10", "37", "12", "76", "97", "18", "44", "66", "64", "47", "72", "63", "79", "52", "3", "80", "5", "25", "57", "73", "42", "35", "38", "7", "1", "88", "84", "20", "28", "2", "33", "23", "70", "24", "0", "74", "58", "61", "83", "40", "68", "6", "55", "34", "48", "78", "27", "14", "49", "94", "75", "11"]}, "24": {"user_id": "46", "name": "Jack Sparrow", "followers": 4255, "bio": "Sailing the seven seas", "posts": "Just found a hidden treasure map.", "friends": ["76", "9", "87", "42", "18", "55", "28", "56", "20", "45", "32", "15", "16", "97", "0", "51", "35", "81", "31", "99", "96", "86", "2", "36", "67", "43", "37", "60", "71", "59", "11", "90", "83", "44", "75", "54", "27", "94", "30", "48", "84", "25", "39", "47", "61", "91", "12", "49", "24", "53", "78", "69", "92", "66", "1", "88", "21", "93", "14"]}, "25": {"user_id": "47", "name": "Emma Green", "followers": 1895, "bio": "Photography", "posts": "Uploaded a beautiful snap of a cat just now.", "friends": ["54", "28", "77", "20", "87", "6", "60", "61", "95", "7", "34", "76", "79", "85", "51", "53", "99", "58", "41", "75", "25", "48", "45", "42", "8", "0", "90", "32", "21", "33", "70", "15", "82", "14", "3", "23", "17", "46", "30", "72"]}, "26": {"user_id": "49", "name": "user", "followers": 3642, "bio": "profile Generator", "posts": "Users", "friends": ["81", "59", "83", "76", "97", "32", "34", "40", "6", "62", "12", "20", "8", "89", "21", "2", "9", "91", "65", "19", "38", "16", "64", "30", "43", "69", "71", "15", "50", "44", "95"]}, "27": {"user_id": "50", "name": "John Doe", "followers": 1585, "bio": "Cooking and photography", "posts": "Uploaded a mouth-watering dish this morning.", "friends": ["4", "33", "45", "48", "10", "29", "56", "69", "41", "11", "5", "73", "77", "12", "63", "7", "16", "13", "0", "94", "91", "81", "83", "2", "55", "62", "92", "6", "65", "20", "1", "76", "89", "90", "58", "49", "96", "57", "71", "32", "31", "30", "38", "68", "97", "42", "54", "17", "8", "86", "79", "24", "82", "99", "25", "43", "22", "80", "3", "9", "72", "37", "26"]}, "28": {"user_id": "51", "name": "Max Johnson", "followers": 2645, "bio": "Avid reader and book reviewer", "posts": "Just started", "friends": ["54", "73", "25", "55", "34", "70", "97", "1", "88", "33", "91", "17", "16", "67", "0", "45", "56", "50", "98", "79", "13", "95", "14", "90", "18", "22", "23", "6", "8", "66", "10", "19", "15", "68", "74", "36", "27", "86", "39", "42", "57", "38", "96", "80", "72", "30", "41", "49", "81", "46", "77", "2", "21", "69", "32", "59", "89", "84", "3", "87", "82", "7", "48", "85", "28", "20", "99", "92", "44", "63", "58", "78"]}, "29": {"user_id": "57", "name": "Bob Johnson", "followers": 1248, "bio": "Tech Enthusiast", "posts": "Recently upgraded my home office setup.", "friends": ["41", "16", "27", "5", "3", "47"]}, "30": {"user_id": "60", "name": "", "followers": 1460, "bio": "Alice at", "posts": "Alice,", "friends": ["85", "52", "20", "90", "54", "36", "50", "49", "44", "88", "6", "7", "53", "89", "42", "83", "35", "51", "57", "77", "76", "48", "92", "15", "72", "61", "26", "82", "19", "87", "96", "95", "84", "17", "80", "23", "3", "11", "98", "29", "47", "67", "81", "66"]}, "31": {"user_id": "63", "name": "James Bond", "followers": 4396, "bio": "Spy gear enthusiast!", "posts": "Just upgraded to the latest night vision goggles.", "friends": ["70", "2", "38", "97", "25", "92", "10", "64", "20", "11", "58", "17", "82", "35", "62", "67", "72", "68", "50", "75", "1", "53", "54", "39", "89", "21", "34", "15", "7", "51", "45", "48", "84", "14", "22", "93", "69", "42", "49", "9", "94", "27", "73", "86", "81", "18", "87", "44", "74", "65", "40", "78", "57", "80", "30", "55", "85", "4", "0", "5", "99", "32", "88", "52", "6", "3", "16", "28", "91", "31", "23", "33", "71", "77", "96", "83", "66", "60", "19", "59", "79", "13", "37", "24", "56", "8", "95", "41", "98", "61", "47", "43"]}, "32": {"user_id": "70", "name": "Jordan Baker", "followers": 1003, "bio": "Classic literature and art", "posts": "Just finished reading", "friends": ["72", "73", "24", "90", "51", "64", "33", "86", "30", "97", "8", "28", "35", "61", "77", "16", "63", "4", "10", "59", "13", "85", "18", "49", "80", "53", "76", "6", "96", "81", "84", "41", "21", "25", "88", "9", "5", "69", "62", "40", "74", "37", "12", "14", "95", "91", "23", "55", "38", "46", "94", "3", "19", "42", "29", "50", "75", "45", "83", "31", "89", "58", "20", "15", "66", "0", "57", "93", "54", "60", "1", "48", "79", "82", "68", "56", "78", "39"]}, "33": {"user_id": "71", "name": "Alice Wonderland", "followers": 3830, "bio": "Exploring the world one frame at a time!", "posts": "Just captured a stunning sunset.", "friends": ["82", "54", "9", "46", "81", "35", "41", "12", "1", "14", "87", "6", "90", "86", "3", "36", "20", "19", "40", "38", "34", "7", "73", "63", "2", "56", "13", "50", "21", "32", "84", "23", "17", "92", "67", "94", "65", "4", "83", "95", "43", "48", "66", "62", "98", "49", "80", "88", "70", "55", "72", "96", "52", "75", "26", "15", "30", "37", "16", "51", "91", "24", "99", "58", "47", "44", "11", "8", "45", "76", "10", "59", "18", "27", "29", "93", "74", "25", "68", "79", "61", "0", "69", "39", "53", "60", "28", "64", "97", "78"]}, "34": {"user_id": "75", "name": "Alice Wonderland", "followers": 3446, "bio": "Exploring the world one frame at a time!", "posts": "Just captured a stunning sunset.", "friends": ["81", "85", "72", "53", "41", "70", "16", "30", "95", "62", "45", "78", "25", "68", "38", "90", "8", "83", "9", "50", "55", "20", "65", "60", "3", "42", "48", "66", "4", "6", "98", "97", "77", "34", "29", "47", "89", "84", "28", "7", "59", "1", "57", "0", "21", "76", "63", "69", "82", "27", "93", "22", "61", "35", "10", "58", "23", "39", "37", "92", "14", "11", "67", "71", "18", "99", "40", "33", "96", "36", "44", "79", "32", "52", "56", "80", "43", "86", "19", "31", "2", "73", "46", "26", "87", "51", "64", "24", "94", "17", "88"]}, "35": {"user_id": "79", "name": "Jack Thompson", "followers": 2433, "bio": "Avid gamer and tech enthusiast", "posts": "Recently unlocked a new achievement in Cyberpunk", "friends": ["83", "13", "17", "12", "88", "62", "39", "72", "91", "86", "7", "67", "22", "69", "15", "42", "98", "24", "34", "71", "32", "49", "70", "6", "5", "82", "21", "73", "50", "3", "23", "28", "77", "35", "90", "84", "37", "96", "63", "89", "19", "40", "45", "74", "10", "51", "92", "53", "60", "59", "44", "26", "25", "68", "4", "65", "41", "18", "8", "1", "31", "52", "56", "87", "29", "2", "58", "64", "75", "0", "16", "66", "97", "14"]}, "36": {"user_id": "83", "name": "Bob Johnson", "followers": 2052, "bio": "Tech Gadgets & Sci-Fi", "posts": "Just posted about the new iPhone.", "friends": ["57", "47", "50", "98", "13", "69", "74", "25", "88", "3", "77", "0", "99", "53", "48", "80", "72", "42", "30", "19", "84", "62", "5", "23", "33", "17", "65", "56", "2", "28", "94", "96", "26", "90", "37", "87", "91", "67", "32", "43", "21", "31", "44"]}, "37": {"user_id": "84", "name": "Alex Morgan", "followers": 2847, "bio": "Soccer enthusiast and a passionate traveler", "posts": "Just finished reading", "friends": ["51", "6", "90", "36", "87", "20", "4", "41", "47", "91", "74", "29", "89", "97", "21", "37", "31", "28", "72", "5", "23", "82", "49", "65", "66", "95", "68", "83", "59", "22", "30", "15", "86", "92", "50", "99", "44", "40", "25", "85", "42", "80", "11", "73", "67", "10", "57", "17", "48", "56", "94", "7", "63", "27", "78", "64", "98", "79", "77", "58", "76", "45", "16", "61", "39", "3", "88", "32", "62", "12", "69", "38", "18", "96", "19", "0", "35", "26", "13", "52", "54"]}, "38": {"user_id": "86", "name": "", "followers": 4094, "bio": "Loves animals and hiking", "posts": "Uploaded a picture with a new furry friend today.", "friends": ["46", "82", "88", "94", "59", "10", "15", "70", "44", "52", "28", "11", "61", "48", "41", "83", "84", "2", "89", "68", "4", "7", "95", "80", "37", "22", "25", "47", "97", "62", "9", "64", "79", "50", "20", "38", "78", "32", "92", "81", "26", "77", "51", "76", "17", "33", "13", "56", "42", "36", "40", "24", "21", "66", "63", "30", "96", "90", "23", "34", "65", "14", "55", "99", "49", "73", "53", "29", "16", "45", "60", "0", "39", "8", "1", "93", "74", "58", "3"]}, "39": {"user_id": "96", "name": "Jack Black", "followers": 2600, "bio": "Rocking out on guitar", "posts": "Just added a new cover song to my profile.", "friends": ["24", "58", "51", "2", "8", "21", "73", "28", "25", "85", "97", "4", "75", "80", "10", "65", "44", "34", "60", "27", "68", "82", "12", "91", "40", "79", "69", "3", "66", "92", "50", "15", "70", "87", "46", "41", "33", "90", "99", "94", "78", "31", "36", "81", "59", "23", "71", "53", "1", "30", "9", "57", "93", "95", "39", "6", "62", "26", "45", "32", "42", "19", "29", "38", "35", "86", "5", "88", "67"]}, "40": {"user_id": "97", "name": "Alex Johnson", "followers": 4712, "bio": "Tech Enthusiast", "posts": "Just finished building a custom PC.", "friends": ["27", "46", "49", "28", "17", "16", "15", "8", "24", "70", "35", "32", "20", "22", "86", "90", "72", "82", "45", "25", "11", "1", "55", "63", "71", "75", "77", "44", "13", "51", "94", "9", "53", "31", "84", "95", "3", "12", "73", "83", "81", "23", "93", "40", "33", "36", "30", "76", "0", "21", "58", "38"]}, "41": {"user_id": "99", "name": "John Doe", "followers": 906, "bio": "Hiking & Photography", "posts": "Uploaded a picture from my latest trail adventure.", "friends": ["22", "85", "2", "62", "21", "76", "98", "80", "77", "19", "3", "28", "35", "67", "39", "25", "10", "75", "63", "78", "93", "0", "14", "83", "42", "91", "45", "44", "86", "79", "4", "87", "64", "11", "41", "89", "12", "48", "94", "58", "31", "90", "84", "73", "66", "74"]}}}
\ No newline at end of file
diff --git a/app/generate_data.py b/app/database/generate_data.py
similarity index 87%
rename from app/generate_data.py
rename to app/database/generate_data.py
index 14ad586..adc0342 100644
--- a/app/generate_data.py
+++ b/app/database/generate_data.py
@@ -5,6 +5,7 @@ import warnings
warnings.filterwarnings('ignore')
import re
import random
+import json
HUGGINGFACEHUB_API_TOKEN = None
os.environ["HUGGINGFACEHUB_API_TOKEN"] = HUGGINGFACEHUB_API_TOKEN
@@ -56,11 +57,20 @@ def generate_data(num_users):
huggingfacehub_api_token=HUGGINGFACEHUB_API_TOKEN,
)
llm_chain = prompt | llm
- data = []
+ data = {}
for i in range(num_users):
raw_text = llm_chain.invoke({"user_id": i})
user_profile = parse_profile(raw_text, i, num_users)
if user_profile:
- data.append(user_profile)
+ data[i] = user_profile
- return data
\ No newline at end of file
+ return data
+
+if __name__ == "__main__":
+ data = generate_data(100)
+
+ # Create json file
+ json_object = json.dumps( {"_default": data}, indent=4 )
+ with open( "datastore/llmData_sns.json", "w" ) as f:
+ f.write( json_object )
+
\ No newline at end of file