update

app.py

@@ -7,6 +7,7 @@ import string
 import logging
 from datetime import datetime
 from huggingface_hub import login, HfApi, hf_hub_download
+from statistics import mean
 
 # Set up logging
 logging.basicConfig(level=logging.INFO,
@@ -25,48 +26,47 @@ else:
     logger.error("HF_TOKEN not found in environment variables")
 
 app = Flask(__name__)
-app.config['SECRET_KEY'] = 'supersecretkey'  # Change this to a random secret key
+app.config['SECRET_KEY'] = 'supersecretkey'
 
 # File-based session storage
 SESSION_DIR = '/tmp/sessions'
 os.makedirs(SESSION_DIR, exist_ok=True)
 
-# Update the VISUALIZATION_DIRS dictionary
+# Update visualization directories for the 4 methods
 VISUALIZATION_DIRS = {
-    "No-XAI": "htmls_NO_XAI_mod",
+    "Text2SQL": "htmls_Text2SQL",
     "Dater": "htmls_DATER_mod2",
     "Chain-of-Table": "htmls_COT_mod",
-    "Plan-of-SQLs": "htmls_POS_mod2",
-    "Text2SQL": "htmls_Text2SQL"
+    "Plan-of-SQLs": "htmls_POS_mod2"
 }
 
-# Update the get_method_dir function
+
+# Update method directory mapping
 def get_method_dir(method):
-    if method == 'No-XAI':
-        return 'NO_XAI'
-    elif method == 'Dater':
-        return 'DATER'
-    elif method == 'Chain-of-Table':
-        return 'COT'
-    elif method == 'Plan-of-SQLs':
-        return 'POS'
-    elif method == 'Text2SQL':
-        return 'Text2SQL'
-    else:
-        return None
-
-# Update the METHODS list
-METHODS = ["No-XAI", "Dater", "Chain-of-Table", "Plan-of-SQLs", "Text2SQL"]
+    method_mapping = {
+        'Text2SQL': 'Text2SQL',
+        'Dater': 'DATER',
+        'Chain-of-Table': 'COT',
+        'Plan-of-SQLs': 'POS'
+    }
+    return method_mapping.get(method)
+
+
+# Update methods list to only include the 4 methods we want to rank
+METHODS = ["Text2SQL", "Dater", "Chain-of-Table", "Plan-of-SQLs"]
+
 
 def generate_session_id():
     return str(uuid.uuid4())
 
+
 def save_session_data(session_id, data):
     file_path = os.path.join(SESSION_DIR, f'{session_id}.json')
     with open(file_path, 'w') as f:
         json.dump(data, f)
     logger.info(f"Session data saved for session {session_id}")
 
+
 def load_session_data(session_id):
     file_path = os.path.join(SESSION_DIR, f'{session_id}.json')
     if os.path.exists(file_path):
@@ -74,6 +74,7 @@ def load_session_data(session_id):
             return json.load(f)
     return None
 
+
 def save_session_data_to_hf(session_id, data):
     try:
         username = data.get('username', 'unknown')
@@ -88,7 +89,7 @@ def save_session_data_to_hf(session_id, data):
             f.write(json_data)
 
         api = HfApi()
-        repo_path = "session_data_foward_simulation"
+        repo_path = "session_data_preference_ranking"
 
         api.upload_file(
             path_or_fileobj=temp_file_path,
@@ -100,229 +101,156 @@ def save_session_data_to_hf(session_id, data):
         logger.info(f"Session data saved for session {session_id} in Hugging Face Data Space")
     except Exception as e:
         logger.exception(f"Error saving session data for session {session_id}: {e}")
-#
-# def load_samples():
-#     common_samples = []
-#     categories = ["TP", "TN", "FP", "FN"]
-#
-#     for category in categories:
-#         files = set(os.listdir(f'htmls_NO_XAI_mod/{category}'))
-#         for method in ["Dater", "Chain-of-Table", "Plan-of-SQLs", "Text2SQL"]:
-#             method_dir = VISUALIZATION_DIRS[method]
-#             files &= set(os.listdir(f'{method_dir}/{category}'))
-#
-#         for file in files:
-#             common_samples.append({'category': category, 'file': file})
-#
-#     logger.info(f"Found {len(common_samples)} common samples across all methods")
-#     return common_samples
-
-def load_samples(method, metadata):
+
+
+def load_samples_for_all_methods(metadata_files):
+    samples_by_method = {}
     common_samples = []
-    categories = ["TP", "TN", "FP", "FN"]
-
-    for category in categories:
-        # files = set(os.listdir(f'htmls_NO_XAI_mod/{category}'))
-        method_dir = VISUALIZATION_DIRS[method]
-        files = set(os.listdir(f'{method_dir}/{category}'))
-
-        for file in files:
-            index = file.split('-')[1].split('.')[0]
-            metadata_key = f"{get_method_dir(method)}_test-{index}.html"
-            sample_metadata = metadata.get(metadata_key, {})
-
-            common_samples.append({
-                'category': category,
-                'file': file,
-                'metadata': sample_metadata
-            })
 
-    logger.info(f"Found {len(common_samples)} samples for method {method}")
+    # First, load all samples for each method
+    for method in METHODS:
+        method_samples = []
+        categories = ["TP", "TN", "FP", "FN"]
+
+        for category in categories:
+            method_dir = VISUALIZATION_DIRS[method]
+            try:
+                files = set(os.listdir(f'{method_dir}/{category}'))
+
+                for file in files:
+                    index = file.split('-')[1].split('.')[0]
+                    metadata_key = f"{get_method_dir(method)}_test-{index}.html"
+
+                    # Get metadata for this sample
+                    sample_metadata = metadata_files[method].get(metadata_key, {})
+
+                    method_samples.append({
+                        'category': category,
+                        'file': file,
+                        'metadata': sample_metadata
+                    })
+            except Exception as e:
+                logger.error(f"Error loading samples for method {method}, category {category}: {e}")
+
+        samples_by_method[method] = method_samples
+
+    # Find common samples across all methods
+    file_sets = []
+    for method, samples in samples_by_method.items():
+        file_set = {s['file'] for s in samples}
+        file_sets.append(file_set)
+
+    common_files = set.intersection(*file_sets)
+
+    # Create groups of samples that exist across all methods
+    for file_name in common_files:
+        sample_group = {}
+        for method in METHODS:
+            sample = next((s for s in samples_by_method[method] if s['file'] == file_name), None)
+            if sample:
+                sample_group[method] = sample
+        if len(sample_group) == len(METHODS):
+            common_samples.append(sample_group)
+
     return common_samples
 
+
 def select_balanced_samples(samples):
     try:
-        # Separate samples into two groups
-        tp_fp_samples = [s for s in samples if s['category'] in ['TP', 'FP']]
-        tn_fn_samples = [s for s in samples if s['category'] in ['TN', 'FN']]
+        # Get the category from any method (they should all be the same)
+        sample_categories = [(s, next(iter(s.values()))['category']) for s in samples]
 
-        # Check if we have enough samples in each group
-        if len(tp_fp_samples) < 5 or len(tn_fn_samples) < 5:
-            logger.warning(f"Not enough samples in each category. TP+FP: {len(tp_fp_samples)}, TN+FN: {len(tn_fn_samples)}")
-            return samples if len(samples) <= 10 else random.sample(samples, 10)
-
-        # Select 5 samples from each group
-        selected_tp_fp = random.sample(tp_fp_samples, 5)
-        selected_tn_fn = random.sample(tn_fn_samples, 5)
-
-        # Combine and shuffle the selected samples
-        selected_samples = selected_tp_fp + selected_tn_fn
-        random.shuffle(selected_samples)
+        # Separate samples into two groups
+        tp_fp_samples = [s for s, cat in sample_categories if cat in ['TP', 'FP']]
+        tn_fn_samples = [s for s, cat in sample_categories if cat in ['TN', 'FN']]
+
+        # Select balanced samples
+        if len(tp_fp_samples) >= 5 and len(tn_fn_samples) >= 5:
+            selected_tp_fp = random.sample(tp_fp_samples, 5)
+            selected_tn_fn = random.sample(tn_fn_samples, 5)
+            selected_samples = selected_tp_fp + selected_tn_fn
+            random.shuffle(selected_samples)
+        else:
+            logger.warning(
+                f"Not enough samples for balanced selection. TP+FP: {len(tp_fp_samples)}, TN+FN: {len(tn_fn_samples)}")
+            selected_samples = random.sample(samples, min(10, len(samples)))
 
-        logger.info(f"Selected 10 balanced samples: 5 from TP+FP, 5 from TN+FN")
         return selected_samples
     except Exception as e:
         logger.exception("Error selecting balanced samples")
         return []
 
-# @app.route('/')
-# def introduction():
-#     return render_template('introduction.html')
+
+@app.route('/')
+def root():
+    return redirect(url_for('consent'))
+
+
+@app.route('/consent', methods=['GET', 'POST'])
+def consent():
+    if request.method == 'POST':
+        return redirect(url_for('introduction'))
+    return render_template('consent.html')
+
+
+@app.route('/introduction')
+def introduction():
+    return render_template('introduction.html')
+
 
 @app.route('/attribution')
 def attribution():
     return render_template('attribution.html')
-#
-# @app.route('/index', methods=['GET', 'POST'])
-# def index():
-#     if request.method == 'POST':
-#         username = request.form.get('username')
-#         seed = request.form.get('seed')
-#         method = request.form.get('method')
-#         if not username or not seed or not method:
-#             return render_template('index.html', error="Please fill in all fields and select a method.")
-#         if method not in ['Chain-of-Table', 'Plan-of-SQLs', 'Dater', 'Text2SQL']:
-#             return render_template('index.html', error="Invalid method selected.")
-#         try:
-#             seed = int(seed)
-#             random.seed(seed)
-#             all_samples = load_samples()
-#             selected_samples = select_balanced_samples(all_samples)
-#             if len(selected_samples) == 0:
-#                 return render_template('index.html', error="No common samples were found")
-#             start_time = datetime.now().isoformat()
-#             session_id = generate_session_id()
-#             session_data = {
-#                 'username': username,
-#                 'seed': str(seed),
-#                 'method': method,
-#                 'selected_samples': selected_samples,
-#                 'current_index': 0,
-#                 'responses': [],
-#                 'start_time': start_time,
-#                 'session_id': session_id
-#             }
-#             save_session_data(session_id, session_data)
-#             logger.info(f"Session data stored for user {username}, method {method}, session_id {session_id}")
-#
-#             # Redirect to explanation for all methods
-#             return redirect(url_for('explanation', session_id=session_id))
-#         except Exception as e:
-#             logger.exception(f"Error in index route: {e}")
-#             return render_template('index.html', error="An error occurred. Please try again.")
-#     return render_template('index.html', show_no_xai=False)
+
 
 @app.route('/index', methods=['GET', 'POST'])
 def index():
     if request.method == 'POST':
         username = request.form.get('username')
         seed = request.form.get('seed')
-        method = request.form.get('method')
-        if not username or not seed or not method:
-            return render_template('index.html', error="Please fill in all fields and select a method.")
-        if method not in ['Chain-of-Table', 'Plan-of-SQLs', 'Dater', 'Text2SQL', 'No-XAI']:
-            return render_template('index.html', error="Invalid method selected.")
+
+        if not username or not seed:
+            return render_template('index.html', error="Please fill in all fields.")
+
         try:
             seed = int(seed)
             random.seed(seed)
 
-            # Load the appropriate metadata file
-            if method == "Chain-of-Table":
-                json_file = 'Tabular_LLMs_human_study_vis_6_COT.json'
-            elif method == "Plan-of-SQLs":
-                json_file = 'Tabular_LLMs_human_study_vis_6_POS.json'
-            elif method == "Dater":
-                json_file = 'Tabular_LLMs_human_study_vis_6_DATER.json'
-            elif method == "No-XAI":
-                json_file = 'Tabular_LLMs_human_study_vis_6_NO_XAI.json'
-            elif method == "Text2SQL":
-                json_file = 'Tabular_LLMs_human_study_vis_6_Text2SQL.json'
-
-            with open(json_file, 'r') as f:
-                metadata = json.load(f)
-
-            all_samples = load_samples(method, metadata)
+            # Load metadata for all methods
+            metadata_files = {}
+            for method in METHODS:
+                json_file = f'Tabular_LLMs_human_study_vis_6_{get_method_dir(method)}.json'
+                with open(json_file, 'r') as f:
+                    metadata_files[method] = json.load(f)
+
+            # Load and select samples
+            all_samples = load_samples_for_all_methods(metadata_files)
             selected_samples = select_balanced_samples(all_samples)
+
             if len(selected_samples) == 0:
                 return render_template('index.html', error="No common samples were found")
-            start_time = datetime.now().isoformat()
+
+            # Create session
             session_id = generate_session_id()
             session_data = {
                 'username': username,
                 'seed': str(seed),
-                'method': method,
                 'selected_samples': selected_samples,
                 'current_index': 0,
                 'responses': [],
-                'start_time': start_time,
+                'start_time': datetime.now().isoformat(),
                 'session_id': session_id
             }
             save_session_data(session_id, session_data)
-            logger.info(f"Session data stored for user {username}, method {method}, session_id {session_id}")
 
-            # Redirect to explanation for all methods
-            return redirect(url_for('explanation', session_id=session_id))
+            return redirect(url_for('experiment', session_id=session_id))
+
         except Exception as e:
             logger.exception(f"Error in index route: {e}")
             return render_template('index.html', error="An error occurred. Please try again.")
-    return render_template('index.html', show_no_xai=False)
-
-@app.route('/explanation/<session_id>')
-def explanation(session_id):
-    session_data = load_session_data(session_id)
-    if not session_data:
-        logger.error(f"No session data found for session ID: {session_id}")
-        return redirect(url_for('index'))
-
-    method = session_data.get('method')
-    if not method:
-        logger.error(f"No method found in session data for session ID: {session_id}")
-        return redirect(url_for('index'))
-
-    if method == 'Chain-of-Table':
-        return render_template('cot_intro.html', session_id=session_id)
-    elif method == 'Plan-of-SQLs':
-        return render_template('pos_intro.html', session_id=session_id)
-    elif method == 'Dater':
-        return render_template('dater_intro.html', session_id=session_id)
-    elif method == 'Text2SQL':
-        return render_template('text2sql_intro.html', session_id=session_id)
-    else:
-        logger.error(f"Invalid method '{method}' for session ID: {session_id}")
-        return redirect(url_for('index'))
-#
-# @app.route('/experiment/<session_id>', methods=['GET', 'POST'])
-# def experiment(session_id):
-#     try:
-#         session_data = load_session_data(session_id)
-#         if not session_data:
-#             return redirect(url_for('index'))
-#
-#         selected_samples = session_data['selected_samples']
-#
-#         method = session_data['method']
-#         current_index = session_data['current_index']
-#
-#         if current_index >= len(selected_samples):
-#             return redirect(url_for('completed', session_id=session_id))
-#
-#         sample = selected_samples[current_index]
-#         visualization_dir = VISUALIZATION_DIRS[method]
-#         visualization_path = f"{visualization_dir}/{sample['category']}/{sample['file']}"
-#
-#         statement = """
-# Please note that in select row function, starting index is 0 for Chain-of-Table and 1 for Dater and Index * represents the selection for all rows.
-#         """
-#
-#         return render_template('experiment.html',
-#                                sample_id=current_index,
-#                                statement=statement,
-#                                visualization=url_for('send_visualization', filename=visualization_path),
-#                                session_id=session_id,
-#                                method=method)
-#     except Exception as e:
-#         logger.exception(f"An error occurred in the experiment route: {e}")
-#         return "An error occurred", 500
+
+    return render_template('index.html')
+
 
 @app.route('/experiment/<session_id>', methods=['GET', 'POST'])
 def experiment(session_id):
@@ -332,200 +260,106 @@ def experiment(session_id):
             return redirect(url_for('index'))
 
         selected_samples = session_data['selected_samples']
-        method = session_data['method']
         current_index = session_data['current_index']
 
         if current_index >= len(selected_samples):
             return redirect(url_for('completed', session_id=session_id))
 
-        sample = selected_samples[current_index]
-        visualization_dir = VISUALIZATION_DIRS[method]
-        visualization_path = f"{visualization_dir}/{sample['category']}/{sample['file']}"
+        if request.method == 'POST':
+            # Validate and save rankings
+            rankings = {method: int(request.form.get(method)) for method in METHODS}
+
+            if not all(1 <= rank <= 4 for rank in rankings.values()):
+                return "Invalid rankings. Please use numbers 1-4.", 400
+            if len(set(rankings.values())) != 4:
+                return "Each method must have a unique rank.", 400
 
-        # Extract metadata
-        metadata = sample.get('metadata', {})
+            session_data['responses'].append({
+                'sample_id': current_index,
+                'rankings': rankings
+            })
+            session_data['current_index'] += 1
+            save_session_data(session_id, session_data)
+            return redirect(url_for('experiment', session_id=session_id))
 
-        # Log the metadata
-        logger.info(f"Sample metadata for session {session_id}, method {method}, index {current_index}: {metadata}")
+        # Get current sample group and prepare visualizations
+        sample_group = selected_samples[current_index]
+        visualizations = {
+            method: url_for('send_visualization',
+                            filename=f"{VISUALIZATION_DIRS[method]}/{sample['category']}/{sample['file']}")
+            for method, sample in sample_group.items()
+        }
 
-        statement = metadata['statement']
-        if method == 'Text2SQL':
-            statement = f""
+        # Get metadata from any method (they should all have the same statement)
+        sample_metadata = next(iter(sample_group.values()))['metadata']
+        statement = sample_metadata.get('statement', '')
 
         return render_template('experiment.html',
                                sample_id=current_index,
                                statement=statement,
-                               visualization=url_for('send_visualization', filename=visualization_path),
-                               session_id=session_id,
-                               method=method,
-                               metadata=metadata)  # Pass metadata to the template
+                               visualizations=visualizations,
+                               methods=METHODS,
+                               session_id=session_id)
+
     except Exception as e:
         logger.exception(f"An error occurred in the experiment route: {e}")
         return "An error occurred", 500
 
-@app.route('/')
-def root():
-    return redirect(url_for('consent'))
 
-@app.route('/consent', methods=['GET', 'POST'])
-def consent():
-    if request.method == 'POST':
-        # User has agreed to the consent
-        return redirect(url_for('introduction'))
-    return render_template('consent.html')
-
-@app.route('/introduction')
-def introduction():
-    return render_template('introduction.html')
-
-@app.route('/subjective/<session_id>', methods=['GET', 'POST'])
-def subjective(session_id):
-    if request.method == 'POST':
-        understanding = request.form.get('understanding')
-
-        session_data = load_session_data(session_id)
-        if not session_data:
-            logger.error(f"No session data found for session: {session_id}")
-            return redirect(url_for('index'))
-
-        session_data['subjective_feedback'] = understanding
-        save_session_data(session_id, session_data)
-
-        return redirect(url_for('completed', session_id=session_id))
-
-    return render_template('subjective.html', session_id=session_id)
-
-@app.route('/feedback', methods=['POST'])
-def feedback():
-    try:
-        session_id = request.form['session_id']
-        prediction = request.form['prediction']
-
-        session_data = load_session_data(session_id)
-        if not session_data:
-            logger.error(f"No session data found for session: {session_id}")
-            return redirect(url_for('index'))
-
-        session_data['responses'].append({
-            'sample_id': session_data['current_index'],
-            'user_prediction': prediction
-        })
-
-        session_data['current_index'] += 1
-        save_session_data(session_id, session_data)
-        logger.info(f"Prediction saved for session {session_id}, sample {session_data['current_index'] - 1}")
-
-        if session_data['current_index'] >= len(session_data['selected_samples']):
-            return redirect(url_for('subjective', session_id=session_id))
-
-        return redirect(url_for('experiment', session_id=session_id))
-    except Exception as e:
-        logger.exception(f"Error in feedback route: {e}")
-        return "An error occurred", 500
-
-# Update the completed route to include Text2SQL
 @app.route('/completed/<session_id>')
 def completed(session_id):
     try:
         session_data = load_session_data(session_id)
         if not session_data:
-            logger.error(f"No session data found for session: {session_id}")
             return redirect(url_for('index'))
 
         session_data['end_time'] = datetime.now().isoformat()
         responses = session_data['responses']
-        method = session_data['method']
-
-        if method == "Chain-of-Table":
-            json_file = 'Tabular_LLMs_human_study_vis_6_COT.json'
-        elif method == "Plan-of-SQLs":
-            json_file = 'Tabular_LLMs_human_study_vis_6_POS.json'
-        elif method == "Dater":
-            json_file = 'Tabular_LLMs_human_study_vis_6_DATER.json'
-        elif method == "No-XAI":
-            json_file = 'Tabular_LLMs_human_study_vis_6_NO_XAI.json'
-        elif method == "Text2SQL":
-            json_file = 'Tabular_LLMs_human_study_vis_6_Text2SQL.json'
-        else:
-            return "Invalid method", 400
-
-        with open(json_file, 'r') as f:
-            ground_truth = json.load(f)
-
-        correct_predictions = 0
-        true_predictions = 0
-        false_predictions = 0
-
-        for response in responses:
-            sample_id = response['sample_id']
-            user_prediction = response['user_prediction']
-            visualization_file = session_data['selected_samples'][sample_id]['file']
-            index = visualization_file.split('-')[1].split('.')[0]
-
-            ground_truth_key = f"{get_method_dir(method)}_test-{index}.html"
-            logger.info(f"ground_truth_key: {ground_truth_key}")
-
-            if ground_truth_key in ground_truth:
-                # TODO: Important Note ->
-                # Using model prediction as we are doing forward simulation
-                # Please use ground_truth[ground_truth_key]['answer'].upper() if running verification task
-                model_prediction = ground_truth[ground_truth_key]['prediction'].upper()
-                if user_prediction.upper() == model_prediction:
-                    correct_predictions += 1
 
-                if user_prediction.upper() == "TRUE":
-                    true_predictions += 1
-                elif user_prediction.upper() == "FALSE":
-                    false_predictions += 1
-            else:
-                logger.warning(f"Missing key in ground truth: {ground_truth_key}")
+        # Calculate average ranking for each method
+        average_rankings = {
+            method: mean(r['rankings'][method] for r in responses)
+            for method in METHODS
+        }
 
-        accuracy = (correct_predictions / len(responses)) * 100 if responses else 0
-        accuracy = round(accuracy, 2)
-
-        true_percentage = (true_predictions / len(responses)) * 100 if len(responses) else 0
-        false_percentage = (false_predictions / len(responses)) * 100 if len(responses) else 0
-
-        true_percentage = round(true_percentage, 2)
-        false_percentage = round(false_percentage, 2)
-
-        session_data['accuracy'] = accuracy
-        session_data['true_percentage'] = true_percentage
-        session_data['false_percentage'] = false_percentage
+        # Sort methods by average ranking (ascending)
+        sorted_methods = sorted(
+            average_rankings.items(),
+            key=lambda x: x[1]
+        )
 
-        # Save all the data to Hugging Face at the end
+        session_data['average_rankings'] = average_rankings
         save_session_data_to_hf(session_id, session_data)
 
-        # Remove the local session data file
-        os.remove(os.path.join(SESSION_DIR, f'{session_id}.json'))
+        # Clean up local session file
+        try:
+            os.remove(os.path.join(SESSION_DIR, f'{session_id}.json'))
+        except Exception as e:
+            logger.warning(f"Error removing session file: {e}")
+
+        return render_template(
+            'completed.html',
+            average_rankings=average_rankings,
+            sorted_methods=sorted_methods
+        )
 
-        return render_template('completed.html',
-                               accuracy=accuracy,
-                               true_percentage=true_percentage,
-                               false_percentage=false_percentage)
     except Exception as e:
         logger.exception(f"An error occurred in the completed route: {e}")
         return "An error occurred", 500
 
+
 @app.route('/visualizations/<path:filename>')
 def send_visualization(filename):
-    logger.info(f"Attempting to serve file: {filename}")
     base_dir = os.getcwd()
     file_path = os.path.normpath(os.path.join(base_dir, filename))
     if not file_path.startswith(base_dir):
         return "Access denied", 403
-
     if not os.path.exists(file_path):
         return "File not found", 404
-
     directory = os.path.dirname(file_path)
     file_name = os.path.basename(file_path)
-    logger.info(f"Serving file from directory: {directory}, filename: {file_name}")
     return send_from_directory(directory, file_name)
 
-@app.route('/visualizations/<path:filename>')
-def send_examples(filename):
-    return send_from_directory('', filename)
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=7860, debug=True)

templates/completed.html

@@ -11,17 +11,17 @@
             display: flex;
             justify-content: center;
             align-items: center;
-            height: 100vh;
-            overflow: hidden;
+            min-height: 100vh;
         }
         .container {
-            background-color: rgba(255, 255, 255, 0.9);
+            background-color: rgba(255, 255, 255, 0.95);
             border-radius: 20px;
             padding: 40px;
             box-shadow: 0 10px 30px rgba(0, 0, 0, 0.1);
             text-align: center;
-            max-width: 600px;
+            max-width: 800px;
             width: 90%;
+            margin: 20px;
         }
         h1 {
             color: #2c3e50;
@@ -29,47 +29,47 @@
             margin-bottom: 30px;
             animation: bounce 1s ease;
         }
-        .stats-container {
+        .rankings-container {
             display: flex;
             flex-direction: column;
-            align-items: center;
-            margin-bottom: 30px;
+            gap: 20px;
+            margin: 30px 0;
         }
-        .stat-card {
-            background-color: #fff;
+        .method-card {
+            background-color: white;
             border-radius: 15px;
             padding: 20px;
-            margin: 10px;
-            box-shadow: 0 5px 15px rgba(0, 0, 0, 0.1);
-            width: 80%;
-            transition: transform 0.3s ease;
-        }
-        .prediction-stats {
+            box-shadow: 0 5px 15px rgba(0, 0, 0, 0.05);
             display: flex;
+            align-items: center;
             justify-content: space-between;
-            width: 80%;
-        }
-        .prediction-stat-card {
-            flex: 1;
-            margin: 0 5px;
-        }
-        .stat-card:hover, .prediction-stat-card:hover {
-            transform: translateY(-5px);
+            transition: transform 0.3s ease;
         }
-        .stat-title {
-            font-size: 18px;
-            color: #7f8c8d;
-            margin-bottom: 10px;
+        .method-card:hover {
+            transform: translateY(-3px);
         }
-        .stat-value {
-            font-size: 28px;
+        .method-name {
+            font-size: 1.2em;
             font-weight: bold;
             color: #2c3e50;
         }
+        .average-rank {
+            font-size: 1.5em;
+            font-weight: bold;
+            color: #3498db;
+            background: #f8f9fa;
+            padding: 10px 20px;
+            border-radius: 25px;
+        }
+        .rank-label {
+            font-size: 0.9em;
+            color: #7f8c8d;
+            margin-top: 5px;
+        }
         .button-container {
             margin-top: 30px;
         }
-        button {
+        .home-button {
             background-color: #3498db;
             color: white;
             border: none;
@@ -77,49 +77,62 @@
             font-size: 18px;
             border-radius: 50px;
             cursor: pointer;
+            text-decoration: none;
+            display: inline-block;
             transition: background-color 0.3s ease, transform 0.3s ease;
         }
-        button:hover {
+        .home-button:hover {
             background-color: #2980b9;
             transform: scale(1.05);
         }
+        .summary {
+            background-color: #f8f9fa;
+            padding: 20px;
+            border-radius: 10px;
+            margin-bottom: 30px;
+            text-align: left;
+        }
+        .summary h2 {
+            color: #2c3e50;
+            margin-top: 0;
+        }
         @keyframes bounce {
             0%, 20%, 50%, 80%, 100% {transform: translateY(0);}
             40% {transform: translateY(-30px);}
             60% {transform: translateY(-15px);}
         }
     </style>
-    <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap" rel="stylesheet">
-    <script src="https://cdn.jsdelivr.net/npm/canvas-confetti@1.5.1/dist/confetti.browser.min.js"></script>
 </head>
 <body>
     <div class="container">
-        <h1>Thank you!</h1>
-        <p>You've successfully completed the experiment. Your predictions have been recorded.</p>
-        <div class="stats-container">
-            <div class="stat-card">
-                <div class="stat-title">Your Labeling Accuracy</div>
-                <div class="stat-value">{{ accuracy }}%</div>
-            </div>
-            <div class="prediction-stats">
-                <div class="stat-card prediction-stat-card">
-                    <div class="stat-title">You Predicted TRUE</div>
-                    <div class="stat-value">{{ true_percentage }}%</div>
-                </div>
-                <div class="stat-card prediction-stat-card">
-                    <div class="stat-title">You Predicted FALSE</div>
-                    <div class="stat-value">{{ false_percentage }}%</div>
+        <h1>Thank You!</h1>
+        <p>You've successfully completed all 10 samples. Here are the average rankings for each explanation method:</p>
+
+        <div class="summary">
+            <h2>Ranking Summary</h2>
+            <p>Lower numbers indicate better rankings (1 = best, 4 = worst)</p>
+        </div>
+
+        <div class="rankings-container">
+            {% for method, rank in sorted_methods %}
+            <div class="method-card">
+                <div class="method-info">
+                    <div class="method-name">{{ method }}</div>
+                    <div class="rank-label">Average Ranking</div>
                 </div>
+                <div class="average-rank">{{ "%.2f"|format(rank) }}</div>
             </div>
+            {% endfor %}
         </div>
+
         <div class="button-container">
-            <a href="/" style="text-decoration: none;">
-                <button>Back to Start Page</button>
-            </a>
+            <a href="/" class="home-button">Back to Home</a>
         </div>
     </div>
+
+    <script src="https://cdn.jsdelivr.net/npm/canvas-confetti@1.5.1/dist/confetti.browser.min.js"></script>
     <script>
-        // Trigger confetti animation
+        // Celebration animation
         confetti({
             particleCount: 100,
             spread: 70,

templates/consent.html

@@ -73,8 +73,8 @@
         <h2>Voluntary Participation</h2>
         <p>Your participation in this study is entirely voluntary. You may choose to withdraw at any time without any consequences.</p>
 
-        <h2>Contact Information</h2>
-        <p>If you have any questions or concerns about this study, please contact Anh Nguyen's lab at Auburn CSSE department via anh.ng8@gmail.com.</p>
+<!--        <h2>Contact Information</h2>-->
+<!--        <p>If you have any questions or concerns about this study, please contact Anh Nguyen's lab at Auburn CSSE department via anh.ng8@gmail.com.</p>-->
 
         <p class="highlight">By clicking "I Agree" below, you confirm that you have read and understood this informed consent, and you agree to participate in this TableQA study under the terms described above.</p>
 

templates/experiment.html

@@ -1,153 +1,197 @@
 <!DOCTYPE html>
 <html>
 <head>
-    <title>Experiment</title>
+    <title>Table QA Experiment</title>
     <style>
-        body, html {
-            margin: 0;
-            padding: 0;
-            height: 100%;
+        body {
             font-family: 'Roboto', sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f5f5f5;
         }
         .container {
-            display: flex;
-            flex-direction: column;
-            height: 100vh;
-            width: 100vw;
-            background-color: #ffffff;
-        }
-        .header {
-            padding: 10px;
-            background-color: #f0f0f0;
-            text-align: center;
+            max-width: 1200px;
+            margin: 0 auto;
+            background-color: white;
+            padding: 20px;
+            border-radius: 10px;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
         }
         h1 {
-            margin: 0;
-            font-size: 20px;
-        }
-        .task-description {
-            padding: 10px;
-            background-color: #e0e0e0;
             text-align: center;
+            color: #333;
+            margin-bottom: 30px;
         }
-        .highlight {
-            font-size: 1.1em;
-            font-weight: bold;
-            color: #0056b3;
-            background-color: #e6e6e6;
-            padding: 5px 10px;
-            border-radius: 4px;
-            display: inline-block;
-            margin-top: 5px;
-        }
-        .visualization-container {
-            flex-grow: 1;
-            display: flex;
-            justify-content: center;
-            align-items: center;
+        .progress {
+            text-align: center;
+            font-size: 1.2em;
+            color: #666;
+            margin-bottom: 20px;
+        }
+        .explanations-grid {
+            display: grid;
+            grid-template-columns: repeat(2, 1fr);
+            gap: 20px;
+            margin-bottom: 40px;
+        }
+        .explanation-card {
+            border: 1px solid #ddd;
+            border-radius: 8px;
             overflow: hidden;
         }
+        .explanation-header {
+            background-color: #f8f9fa;
+            padding: 15px;
+            border-bottom: 1px solid #ddd;
+        }
+        .explanation-header h2 {
+            margin: 0;
+            color: #333;
+            font-size: 1.2em;
+        }
+        .explanation-content {
+            padding: 0;
+        }
         iframe {
             width: 100%;
-            height: 100%;
+            height: 500px;
             border: none;
         }
-        .buttons {
-            display: flex;
-            justify-content: space-around;
-            padding: 10px;
-            background-color: #f0f0f0;
+        .ranking-section {
+            position: sticky;
+            bottom: 0;
+            background: white;
+            padding: 20px;
+            border-top: 3px solid #4CAF50;
+            box-shadow: 0 -2px 10px rgba(0,0,0,0.1);
+            margin-top: 40px;
+        }
+        .ranking-grid {
+            display: grid;
+            grid-template-columns: repeat(4, 1fr);
+            gap: 15px;
+            margin: 20px 0;
+        }
+        .ranking-item {
+            background: #f8f9fa;
+            padding: 15px;
+            border-radius: 8px;
+            text-align: center;
+        }
+        .ranking-item label {
+            display: block;
+            margin-bottom: 10px;
+            font-weight: bold;
+            color: #333;
+        }
+        .ranking-item input {
+            width: 60px;
+            padding: 8px;
+            border: 2px solid #ddd;
+            border-radius: 4px;
+            text-align: center;
+            font-size: 1.1em;
         }
-        button {
-            background-color: #808080;
+        .submit-button {
+            background-color: #4CAF50;
             color: white;
-            padding: 10px 20px;
+            padding: 15px 30px;
             border: none;
             border-radius: 5px;
             cursor: pointer;
-            font-size: 16px;
-            transition: background-color 0.3s ease;
-        }
-        button:hover {
-            background-color: #707070;
+            font-size: 1.1em;
+            display: block;
+            margin: 20px auto;
+            transition: background-color 0.3s;
+        }
+        .submit-button:hover {
+            background-color: #45a049;
+        }
+        .instructions {
+            background-color: #e9f5e9;
+            border-left: 4px solid #4CAF50;
+            padding: 15px;
+            margin-bottom: 20px;
+            border-radius: 4px;
         }
-        .bottom-question {
+        .error-message {
+            color: #d32f2f;
             text-align: center;
+            margin: 10px 0;
             padding: 10px;
-            background-color: #e0e0e0;
-            font-weight: bold;
-            font-size: 14px;
-        }
-        /* Loader styles */
-        .overlay {
-            position: fixed;
-            top: 0;
-            left: 0;
-            width: 100%;
-            height: 100%;
-            background-color: rgba(0, 0, 0, 0.5);
-            display: none;
-            z-index: 1000;
-        }
-        .loader {
-            border: 5px solid #f3f3f3;
-            border-top: 5px solid #3498db;
-            border-radius: 50%;
-            width: 50px;
-            height: 50px;
-            animation: spin 1s linear infinite;
-            position: fixed;
-            top: 50%;
-            left: 50%;
-            margin-top: -25px;
-            margin-left: -25px;
+            background-color: #fde8e8;
+            border-radius: 4px;
             display: none;
-            z-index: 1001;
-        }
-        @keyframes spin {
-            0% { transform: rotate(0deg); }
-            100% { transform: rotate(360deg); }
         }
     </style>
-    <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap" rel="stylesheet">
 </head>
 <body>
     <div class="container">
-        <div class="header">
-            <h1>{{ sample_id + 1 }} / 10</h1>
-        </div>
-        <div class="task-description">
-            <p>Please note that in select row function, starting index is 0 for Chain-of-Table and 1 for Dater and Index * represents the selection for all rows.</p>
-            <p class="highlight">Based on the explanation below, please guess what the AI model will predict on the input Statement below.</p>
-            <h2>{{ statement | safe }}</h2>
+        <div class="progress">Sample {{ sample_id + 1 }} of 10</div>
+        
+        <div class="instructions">
+            <h3>Ranking Instructions:</h3>
+            <p>Please examine each explanation method and rank them based on:</p>
+            <ul>
+                <li><strong>Clarity:</strong> How easy is the explanation to understand?</li>
+                <li><strong>Coherence:</strong> Does the explanation logically flow and make sense?</li>
+                <li><strong>Helpfulness:</strong> How well does it reveal the model's reasoning?</li>
+            </ul>
+            <p>Assign ranks from 1 (best) to 4 (worst). Each rank can only be used once.</p>
         </div>
 
-        <div class="visualization-container">
-            <iframe src="{{ visualization }}"></iframe>
-        </div>
-        <div class="bottom-question">
-            <h1>Guess what the model will predict on the Statement based on the provided explanation?</h1>
+        <div class="explanations-grid">
+            {% for method in methods %}
+            <div class="explanation-card">
+                <div class="explanation-header">
+                    <h2>{{ method }}</h2>
+                </div>
+                <div class="explanation-content">
+                    <iframe src="{{ visualizations[method] }}" title="{{ method }}"></iframe>
+                </div>
+            </div>
+            {% endfor %}
         </div>
-        <div class="buttons">
-            <form action="{{ url_for('feedback') }}" method="post" onsubmit="showLoader()">
-                <input type="hidden" name="session_id" value="{{ session_id }}">
-                <button type="submit" name="prediction" value="TRUE"><h1>Model will predict: Statement is TRUE</h1></button>
-            </form>
-            <form action="{{ url_for('feedback') }}" method="post" onsubmit="showLoader()">
-                <input type="hidden" name="session_id" value="{{ session_id }}">
-                <button type="submit" name="prediction" value="FALSE"><h1>Model will predict: Statement is FALSE</h1></button>
+
+        <div class="ranking-section">
+            <form id="rankingForm" action="{{ url_for('experiment', session_id=session_id) }}" method="post" onsubmit="return validateRankings()">
+                <div class="ranking-grid">
+                    {% for method in methods %}
+                    <div class="ranking-item">
+                        <label for="{{ method }}">{{ method }}</label>
+                        <input type="number" id="{{ method }}" name="{{ method }}" min="1" max="4" required>
+                    </div>
+                    {% endfor %}
+                </div>
+                <div id="errorMessage" class="error-message"></div>
+                <button type="submit" class="submit-button">Submit Rankings</button>
             </form>
         </div>
     </div>
 
-    <!-- Loader and overlay -->
-    <div class="overlay" id="overlay"></div>
-    <div class="loader" id="loader"></div>
-
     <script>
-        function showLoader() {
-            document.getElementById('overlay').style.display = 'block';
-            document.getElementById('loader').style.display = 'block';
+        function validateRankings() {
+            const rankings = new Set();
+            const form = document.getElementById('rankingForm');
+            const errorMessage = document.getElementById('errorMessage');
+            
+            for (const input of form.getElementsByTagName('input')) {
+                const value = parseInt(input.value);
+                if (isNaN(value) || value < 1 || value > 4) {
+                    errorMessage.textContent = 'Please use only numbers between 1 and 4.';
+                    errorMessage.style.display = 'block';
+                    return false;
+                }
+                rankings.add(value);
+            }
+
+            if (rankings.size !== 4) {
+                errorMessage.textContent = 'Please assign unique ranks (1-4) to each method.';
+                errorMessage.style.display = 'block';
+                return false;
+            }
+
+            return true;
         }
     </script>
 </body>

templates/index.html

@@ -40,84 +40,6 @@
             border-radius: 5px;
             font-size: 18px;
         }
-        .method-buttons {
-            display: flex;
-            flex-wrap: wrap;
-            justify-content: center;
-            margin-bottom: 20px;
-            gap: 20px;
-        }
-        .method-button {
-            width: calc(45% - 10px);
-            padding: 15px;
-            font-size: 20px;
-            border-radius: 10px;
-            cursor: pointer;
-            transition: all 0.3s ease;
-            border: 2px solid transparent;
-            font-weight: bold;
-            text-align: center;
-        }
-        .method-button.Chain-of-Table {
-            background-color: #ffcc80;
-            color: #1e90ff;
-        }
-        .method-button.Plan-of-SQLs {
-            background-color: #ffcc80;
-            color: #e65100;
-        }
-        .method-button.Text2SQL {
-            background-color: #ffcc80;
-            color: #7b1fa2;
-        }
-        .method-button.Dater {
-            background-color: #ffcc80;
-            color: #4caf50;
-        }
-        .method-button.No-XAI {
-            background-color: #ffcc80;
-            color: #ff9800;
-        }
-        .task-instruction {
-            background-color: #f0f8ff;
-            border-left: 5px solid #4CAF50;
-            padding: 20px;
-            margin-bottom: 30px;
-            border-radius: 5px;
-            text-align: left;
-        }
-        .task-instruction h2 {
-            color: #4CAF50;
-            margin-top: 0;
-        }
-        .task-step {
-            display: flex;
-            align-items: center;
-            margin-bottom: 15px;
-        }
-        .task-icon {
-            font-size: 24px;
-            margin-right: 15px;
-            color: #4CAF50;
-        }
-        .task-text {
-            font-size: 18px;
-            color: #333;
-        }
-        .method-button:hover {
-            opacity: 0.8;
-        }
-        .method-button.selected {
-            border-color: #000000;
-            box-shadow: 0 0 15px rgba(0, 0, 0, 0.3);
-            transform: scale(1.05);
-            animation: borderPulse 0.5s ease-in-out;
-        }
-        @keyframes borderPulse {
-            0% { border-color: transparent; }
-            50% { border-color: #000000; }
-            100% { border-color: #000000; }
-        }
         button {
             background-color: #4CAF50;
             color: white;
@@ -132,121 +54,18 @@
         button:hover {
             background-color: #45a049;
         }
-        .error-message {
-            color: red;
-            margin-bottom: 10px;
-            font-size: 18px;
-        }
     </style>
-    <script>
-        function shuffleArray(array) {
-            for (let i = array.length - 1; i > 0; i--) {
-                const j = Math.floor(Math.random() * (i + 1));
-                [array[i], array[j]] = [array[j], array[i]];
-            }
-            return array;
-        }
-
-        function createMethodButtons() {
-            const methods = [
-                { name: 'Chain-of-Table', color: '#1e90ff' },
-                { name: 'Plan-of-SQLs', color: '#e65100' },
-                { name: 'Text2SQL', color: '#7b1fa2' },
-                { name: 'Dater', color: '#4caf50' },
-                { name: 'No-XAI', color: '#ff9800' }
-            ];
-
-            const shuffledMethods = shuffleArray(methods);
-            const container = document.querySelector('.method-buttons');
-            container.innerHTML = ''; // Clear existing buttons
-
-            shuffledMethods.forEach(method => {
-                if (method.name === 'No-XAI' && !{{ show_no_xai|tojson }}) {
-                    return; // Skip No-XAI if not shown
-                }
-                const button = document.createElement('div');
-                button.className = `method-button ${method.name}`;
-                button.onclick = () => selectMethod(method.name);
-                button.textContent = method.name;
-                container.appendChild(button);
-            });
-        }
-
-        function selectMethod(method) {
-            document.getElementById('method').value = method;
-
-            var buttons = document.getElementsByClassName('method-button');
-            for (var i = 0; i < buttons.length; i++) {
-                buttons[i].classList.remove('selected');
-            }
-
-            var selectedButton = document.querySelector(`.method-button.${method}`);
-            if (selectedButton) {
-                selectedButton.classList.add('selected');
-            }
-        }
-
-        function validateForm() {
-            var username = document.getElementById('username').value;
-            var seed = document.getElementById('seed').value;
-            var method = document.getElementById('method').value;
-            if (!username || !seed || !method) {
-                alert("Please fill in all fields and select a method.");
-                return false;
-            }
-            return true;
-        }
-
-        // Call this function when the page loads
-        window.onload = function() {
-            createMethodButtons();
-        };
-    </script>
     <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap" rel="stylesheet">
 </head>
 <body>
     <div class="container">
-        <div class="task-instruction">
-            <h2>Let's Get Started! 🚀</h2>
-            <div class="task-step">
-                <span class="task-icon">👤</span>
-                <span class="task-text">Enter your name</span>
-            </div>
-            <div class="task-step">
-                <span class="task-icon">🔢</span>
-                <span class="task-text">Choose a lucky number</span>
-            </div>
-            <div class="task-step">
-                <span class="task-icon">📊</span>
-                <span class="task-text">Select an explanation method</span>
-            </div>
-            <div class="task-step">
-                <span class="task-icon">🎯</span>
-                <span class="task-text">Complete 10 samples in the experiment</span>
-            </div>
-        </div>
-
-        {% if error %}
-        <div class="error-message">
-            {{ error }}
-        </div>
-        {% endif %}
-
-        <form id="method-form" action="{{ url_for('index') }}" method="post" onsubmit="return validateForm();">
+        <h1>Trustworthy LLMs for Table QA</h1>
+        <form id="method-form" action="/" method="post">
             <label for="username">Hi there 👋👋👋 ! What is your name?</label>
             <input type="text" id="username" name="username" required>
             <label for="seed">What is your lucky number? 🍀🍀🍀 </label>
             <input type="number" id="seed" name="seed" required>
-
-            <input type="hidden" id="method" name="method" required>
-
-            <h2 style="color: #333; margin-top: 30px; margin-bottom: 20px;">Explanation Methods</h2>
-
-            <div class="method-buttons">
-                <!-- Method buttons will be dynamically inserted here -->
-            </div>
-
-            <button type="submit">Next</button>
+            <button type="submit">Start Experiment</button>
         </form>
     </div>
 </body>

templates/introduction.html

@@ -200,11 +200,18 @@
         </div>
 
         <div class="outro">
-            <h2>Model Simulation Task</h2>
+            <h2>Explanation Ranking Task</h2>
             <p>
-                Given an input statement, an Artificial Intelligence (AI) model will output either TRUE or FALSE.
-                <strong>Your job in this Simulation task is to use the AI's explanation to guess the machine response.</strong>
-                Specifically, please choose which response (Statement is TRUE/ Statement is FALSE) model would output regardless of whether you think that response is correct or not.
+<!--                Given an input statement, an Artificial Intelligence (AI) model will output either TRUE or FALSE.-->
+<!--                <strong>Your job in this Simulation task is to use the AI's explanation to guess the machine response.</strong>-->
+<!--                Specifically, please choose which response (Statement is TRUE/ Statement is FALSE) model would output regardless of whether you think that response is correct or not.-->
+                You are given explanations from <strong>4</strong> different methods for an input.
+                Please rank these explanations based on their clarity, coherence, and helpfulness in understanding the model's reasoning.
+                Clarity Definition: How easy is the explanation to understand? Is the language clear and straightforward?
+                Coherence Definition: Does the explanation logically flow and make sense as a whole? Are the ideas well-connected?
+                Helpfulness in Understanding the Model's Reasoning Definition: How effectively does the explanation help you understand why the model made its decision? Does it reveal the reasoning process?
+
+                Provide the ranking from best to worst (1-best and 4-worst).
             </p>
         </div>