diff --git a/Final Project/animation/animation_engine.py b/Final Project/animation/animation_engine.py new file mode 100644 index 0000000000..7c2692a383 --- /dev/null +++ b/Final Project/animation/animation_engine.py @@ -0,0 +1,1048 @@ +import pygame +import math +import random + +try: + import cv2 + import numpy as np +except ImportError: + cv2 = None + np = None + +from animation.set_profile import get_spectrum_style + + +class AnimationEngine: + def __init__(self, width=1280, height=720): + pygame.init() + self.width = width + self.height = height + self.screen = pygame.display.set_mode((width, height)) + pygame.display.set_caption("Inner Constellation") + self.clock = pygame.time.Clock() + + # Profile / element state + self.current_element = None + self.current_profile = None + self.spectrum_name = "None" + + # Style dictionary from set_profile + self.style = get_spectrum_style([]) # neutral default style + + # Energy scale and temperature + self.scale = 1.0 # base energy size + self.temp_shift = 0.0 # -1.0 (cold) ~ +1.0 (warm) + + # Time and camera-derived energy levels + self.time = 0.0 + self.motion_level = 0.0 + + # Camera motion analysis + self.prev_gray = None + self.downsample_size = (64, 36) + + self.body_x = 0.5 # 0 = left, 1 = right + self.body_y = 0.5 # 0 = top, 1 = bottom + + self.size_level = 0.0 # larger = closer + + self.orbs = [] + self.comets = [] + self.bloom_orbs = [] + + self.stroke_boost = 2 + + # Fallback 单元素颜色 + self.element_colors = { + "Fire": [(255, 120, 60), (255, 200, 90)], + "Water": [(60, 140, 255), (110, 220, 255)], + "Wind": [(190, 230, 255), (150, 210, 255)], + "Earth": [(90, 160, 100), (170, 220, 150)], + "Light": [(255, 255, 255), (255, 240, 210)], + "Shadow": [(120, 70, 160), (60, 30, 100)], + } + + # ------------------------------------------------------------------ + def update(self, profile=None, element=None, gesture=None, proximity=None, frame=None): + """ + Main update entry point. + + Args: + profile: list of 3 elements (or None) + element: single element name (fallback before profile is selected) + gesture: ignored (gesture sensor disabled; camera-only control) + proximity: ignored + frame: OpenCV camera frame (BGR) or None + """ + # Handle window close + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + raise SystemExit + + # 1) Profile / element + if profile is not None and profile != self.current_profile: + self.current_profile = profile + self.current_element = None + self.style = get_spectrum_style(profile) + self.spectrum_name = self.style.get("name", "Spectrum") + self._clear_dynamic_buffers() + print(f"[Animation] Spectrum profile -> {self.spectrum_name}") + + if self.current_profile is None and element is not None and element != self.current_element: + self.current_element = element + self.spectrum_name = element + self.style = get_spectrum_style([element]) + self._clear_dynamic_buffers() + print(f"[Animation] Element -> {element}") + + # 2) + dt_ms = self.clock.get_time() + dt = dt_ms / 1000.0 if dt_ms > 0 else 1.0 / 60.0 + self.time += dt + + # 3) motion + position + size + if frame is not None and cv2 is not None and np is not None: + self._update_motion_features(frame) + + # 4) body_x → temp_shift (-1 ~ +1) + target_temp = (self.body_x - 0.5) * 2.0 + target_temp = max(-1.0, min(1.0, target_temp)) + self.temp_shift = 0.9 * self.temp_shift + 0.1 * target_temp + + # 5) size_level to scale + base_scale = 1.5 + target_scale = base_scale + (0.9 - 1.6 * self.size_level) + self.scale = 0.9 * self.scale + 0.1 * target_scale + self.scale = max(0.6, min(3.0, self.scale)) + + # 6) + breath_from_motion = 1.0 + 0.9 * self.motion_level + breathing_wave = 1.0 + 0.28 * math.sin(self.time * 2.0 * math.pi * 0.4) + self.render_scale = self.scale * breath_from_motion * breathing_wave + + # 7) frame + self._draw_frame(frame, dt) + + pygame.display.flip() + self.clock.tick(60) + + # ------------------------------------------------------------------ + def _clear_dynamic_buffers(self): + self.orbs.clear() + self.comets.clear() + self.bloom_orbs.clear() + + # ------------------------------------------------------------------ + def _update_motion_features(self, frame): + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + gray_small = cv2.resize(gray, self.downsample_size) + + if self.prev_gray is None: + self.prev_gray = gray_small + self.motion_level = 0.0 + self.size_level = 0.0 + return + + diff = cv2.absdiff(gray_small, self.prev_gray) + self.prev_gray = gray_small + + mean_diff = diff.mean() / 255.0 + self.motion_level = 0.85 * self.motion_level + 0.15 * min(1.0, mean_diff * 8.0) + + diff_norm = diff.astype("float32") / 255.0 + mask = diff_norm > 0.18 + active_ratio = mask.mean() + size_raw = min(1.0, active_ratio * 18.0) + self.size_level = 0.9 * self.size_level + 0.1 * size_raw + + total = diff.sum() + if total > 1: + h, w = diff.shape + ys, xs = np.indices((h, w)) + cx = (diff * xs).sum() / total + cy = (diff * ys).sum() / total + + norm_x = float(cx) / max(1.0, w - 1) + norm_y = float(cy) / max(1.0, h - 1) + + self.body_x = 0.85 * self.body_x + 0.15 * norm_x + self.body_y = 0.85 * self.body_y + 0.15 * norm_y + + # ------------------------------------------------------------------ + def _get_body_center(self): + x = int(self.width * (0.15 + 0.7 * self.body_x)) # 15%~85% width + y = int(self.height * (0.25 + 0.5 * self.body_y)) # 25%~75% height + return x, y + + # ------------------------------------------------------------------ + def _draw_frame(self, frame, dt): + # 1) + if self.current_profile is not None or self.current_element is not None: + base_colors = self.style.get( + "base_colors", [(255, 255, 255), (200, 200, 200)] + ) + else: + base_colors = [(255, 255, 255), (200, 200, 200)] + + # 2) night-sky + bg_color = self.style.get("background_color", (5, 7, 18)) + self.screen.fill(bg_color) + + # 3) + temp_t = (self.temp_shift + 1) / 2.0 + temp_t = max(0.0, min(1.0, temp_t)) + + cold_core = (80, 150, 255) + warm_core = (255, 190, 110) + tint_target = self._lerp_color(cold_core, warm_core, temp_t) + + tinted_colors = [] + for c in base_colors: + tinted = self._lerp_color(c, tint_target, 0.65) + boosted = self._boost_color(tinted, factor=1.5) + tinted_colors.append(boosted) + + # 4) + if frame is not None and cv2 is not None: + self._blit_camera(frame) + + # 5) Pattern + pattern_type = self.style.get("pattern_type", "pillar_orbs") + + if pattern_type == "pillar_orbs": + self._pattern_pillar_orbs(tinted_colors, dt) + elif pattern_type == "ring_waves": + self._pattern_ring_waves(tinted_colors) + elif pattern_type == "radial_rays": + self._pattern_radial_rays(tinted_colors) + elif pattern_type == "galaxy": + self._pattern_galaxy(tinted_colors) + elif pattern_type == "double_pillar": + self._pattern_double_pillar(tinted_colors) + elif pattern_type == "vertical_ribbons": + self._pattern_vertical_ribbons(tinted_colors) + elif pattern_type == "grid_pulse": + self._pattern_grid_pulse(tinted_colors) + elif pattern_type == "starfield": + self._pattern_starfield(tinted_colors) + elif pattern_type == "vortex": + self._pattern_vortex(tinted_colors) + elif pattern_type == "cross_waves": + self._pattern_cross_waves(tinted_colors) + elif pattern_type == "aurora": + self._pattern_aurora(tinted_colors) + elif pattern_type == "blooming_orbs": + self._pattern_blooming_orbs(tinted_colors, dt) + elif pattern_type == "comet_trails": + self._pattern_comet_trails(tinted_colors, dt) + elif pattern_type == "spiral_rings": + self._pattern_spiral_rings(tinted_colors) + else: + self._pattern_pillar_orbs(tinted_colors, dt) + + # 6) + self._draw_label() + + # ------------------------------------------------------------------ + # PATTERN 1: central pillar + orbiting orbs + # ------------------------------------------------------------------ + def _pattern_pillar_orbs(self, base_colors, dt): + self._draw_aura_center(base_colors) + self._pattern_orbiting_orbs(base_colors, dt) + + def _draw_aura_center(self, base_colors): + aura_surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + center_x, center_y = self._get_body_center() + + width_factor = self.style.get("pillar_width_factor", 1.0) + height_factor = self.style.get("pillar_height_factor", 1.0) + halo_scale = self.style.get("halo_scale", 1.0) + + pillar_height = int(self.height * 0.7 * self.render_scale * height_factor) + pillar_height = max(int(self.height * 0.4), min(pillar_height, int(self.height * 0.9))) + + base_width = int(220 * self.render_scale * width_factor) + base_width = max(110, min(base_width, int(self.width * 0.8))) + + base_rect = pygame.Rect(0, 0, base_width, pillar_height) + base_rect.centerx = center_x + base_rect.centery = center_y + + wobble = 22 * math.sin(self.time * 2.0 * math.pi * 0.35) + base_rect.centery += int(wobble * (0.7 + 0.5 * self.motion_level)) + + num_layers = 24 + for i in range(num_layers): + t = i / (num_layers - 1) + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(255 * (1.0 - t ** 1.3)) + alpha = int(alpha * (0.9 + 0.9 * self.motion_level)) + alpha = max(0, min(alpha, 255)) + + rgba = (color[0], color[1], color[2], alpha) + + inflate_x = int(base_width * 2.0 * t) + inflate_y = int(pillar_height * 0.5 * t) + layer_rect = base_rect.inflate(inflate_x, inflate_y) + + pygame.draw.ellipse(aura_surface, rgba, layer_rect) + + # + head_y = base_rect.top + int(pillar_height * 0.2) + halo_radius = int(base_width * 1.0 * halo_scale * (0.9 + 0.7 * self.motion_level)) + halo_radius = max(65, halo_radius) + + halo_center = (center_x, head_y) + for i in range(10): + t = i / 9.0 + color = base_colors[min(len(base_colors) - 1, i % len(base_colors))] + alpha = int(250 * (1.0 - t ** 1.8) * (0.8 + 0.6 * self.motion_level)) + radius = int(halo_radius * (0.6 + 0.5 * t * self.render_scale)) + + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + pygame.draw.circle(aura_surface, rgba, halo_center, radius) + + core_color = base_colors[len(base_colors) // 2] + core_rgba = (core_color[0], core_color[1], core_color[2], 255) + core_radius = int(base_width * 0.6 * (1.0 + 0.4 * self.motion_level)) + core_radius = max(45, core_radius) + pygame.draw.circle(aura_surface, core_rgba, halo_center, core_radius) + + self.screen.blit(aura_surface, (0, 0)) + + def _pattern_orbiting_orbs(self, base_colors, dt): + center_x, center_y = self._get_body_center() + + orb_count = self.style.get("orb_count", 36) + r_min, r_max = self.style.get("orb_radius_range", (180, 360)) + speed_min, speed_max = self.style.get("orb_speed_range", (0.6, 1.3)) + size_min, size_max = self.style.get("orb_size_range", (6.0, 14.0)) + vertical_squash = self.style.get("orb_vertical_squash", 0.55) + + if len(self.orbs) < orb_count: + for _ in range(orb_count - len(self.orbs)): + radius = random.uniform(r_min, r_max) * (0.7 + 0.5 * self.render_scale) + angle = random.uniform(0, math.tau) + speed = random.uniform(speed_min, speed_max) + size = random.uniform(size_min, size_max) + color_idx = random.randint(0, len(base_colors) - 1) + self.orbs.append([radius, angle, speed, size, color_idx]) + + for orb in self.orbs: + radius, angle, speed, size, color_idx = orb + + angular_speed = speed * (1.0 + 1.4 * self.motion_level) + orb[1] += angular_speed * dt + + x = center_x + math.cos(orb[1]) * radius + y = center_y + math.sin(orb[1]) * radius * vertical_squash + + final_size = size * (1.1 + 1.0 * self.render_scale) + final_size = max(4.0, min(final_size, 30.0)) + + color = base_colors[color_idx] + pygame.draw.circle(self.screen, color, (int(x), int(y)), int(final_size)) + + # ------------------------------------------------------------------ + # PATTERN 2: ring waves + # ------------------------------------------------------------------ + def _pattern_ring_waves(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + center_x, center_y = self._get_body_center() + + num_rings = 18 + base_radius_y = 70 * self.render_scale + gap = 30 * self.render_scale + + amp = 40 * (0.3 + 0.7 * self.motion_level) + time_factor = self.time * 1.4 + + for i in range(num_rings): + t = i / (num_rings - 1) + radius_y = base_radius_y + i * gap + radius_x = radius_y * 2.0 + + y_offset = math.sin(time_factor + i * 0.35) * amp + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(240 * (1.0 - t ** 1.5)) + alpha = int(alpha * (0.7 + 0.9 * self.motion_level)) + alpha = max(0, min(alpha, 255)) + + rect = pygame.Rect(0, 0, int(radius_x * 2), int(radius_y * 2)) + rect.centerx = center_x + rect.centery = center_y + int(y_offset) + + rgba = (color[0], color[1], color[2], alpha) + ring_width = int(5 * self.stroke_boost) + pygame.draw.ellipse(surface, rgba, rect, width=ring_width) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 3: radial rays + # ------------------------------------------------------------------ + def _pattern_radial_rays(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + center_x, center_y = self._get_body_center() + + num_rays = 24 + inner_radius = 70 * (0.9 + 0.4 * self.render_scale) + outer_base = min(self.width, self.height) * 0.85 * (0.6 + 0.6 * self.render_scale) + + energy = self.motion_level + time_factor = self.time * (1.0 + 1.6 * energy) + + for i in range(num_rays): + t = i / num_rays + angle = t * math.tau + time_factor + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + length = outer_base * ( + 0.7 + + 0.9 * energy + + 0.3 * math.sin(self.time * 2.6 + i * 0.5) + ) + + x1 = center_x + math.cos(angle) * inner_radius + y1 = center_y + math.sin(angle) * inner_radius + x2 = center_x + math.cos(angle) * length + y2 = center_y + math.sin(angle) * length + + alpha = int(255 * (0.55 + 0.7 * energy)) + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + + width = int((5 + 6 * self.render_scale) * self.stroke_boost) + for offset in range(-width // 2, width // 2 + 1): + dx = -math.sin(angle) * offset * 0.8 + dy = math.cos(angle) * offset * 0.8 + start_pos = (int(x1 + dx), int(y1 + dy)) + end_pos = (int(x2 + dx), int(y2 + dy)) + pygame.draw.line(surface, rgba, start_pos, end_pos, 1) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 4: galaxy + # ------------------------------------------------------------------ + def _pattern_galaxy(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + center_x, center_y = self._get_body_center() + + num_arms = 5 + points_per_arm = 80 + base_radius = 70 * self.render_scale + max_radius = min(self.width, self.height) * 0.65 * (0.7 + 0.6 * self.render_scale) + + energy = self.motion_level + spin_speed = 0.6 + 1.2 * energy + + for arm in range(num_arms): + arm_angle = arm * (math.tau / num_arms) + for i in range(points_per_arm): + t = i / (points_per_arm - 1) + radius = base_radius + t * max_radius + + angle = arm_angle + t * 2.5 + self.time * spin_speed + + jitter = (0.5 - random.random()) * 20.0 + x = center_x + math.cos(angle) * radius + jitter + y = center_y + math.sin(angle) * radius * 0.75 + jitter + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(255 * (0.15 + 0.9 * (1.0 - t))) + alpha = int(alpha * (0.4 + 1.0 * energy)) + alpha = max(0, min(alpha, 255)) + + size = 2 + int(4 * (1.0 - t) * (0.8 + 0.9 * self.render_scale)) + + rgba = (color[0], color[1], color[2], alpha) + pygame.draw.circle(surface, rgba, (int(x), int(y)), size) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 5: double pillar + # ------------------------------------------------------------------ + def _pattern_double_pillar(self, base_colors): + aura_surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + + base_cx, cy = self._get_body_center() + spacing = int(self.width * 0.18) + + centers = [ + (base_cx - spacing, cy), + (base_cx + spacing, cy), + ] + + pillar_height = int(self.height * 0.6 * self.render_scale) + pillar_height = max(int(self.height * 0.4), min(pillar_height, int(self.height * 0.9))) + + base_width = int(150 * self.render_scale) + base_width = max(80, min(base_width, int(self.width * 0.45))) + + wobble_amp = 15 * (1.0 + 0.6 * self.motion_level) + + for idx_c, (cx, cy) in enumerate(centers): + base_rect = pygame.Rect(0, 0, base_width, pillar_height) + base_rect.centerx = cx + base_rect.centery = cy + int(wobble_amp * math.sin(self.time * 2.0 + idx_c * 0.7)) + + num_layers = 18 + for i in range(num_layers): + t = i / (num_layers - 1) + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(240 * (1.0 - t ** 1.4) * (0.7 + 0.9 * self.motion_level)) + alpha = max(0, min(alpha, 255)) + rgba = (color[0], color[1], color[2], alpha) + + inflate_x = int(base_width * 1.5 * t) + inflate_y = int(pillar_height * 0.5 * t) + layer_rect = base_rect.inflate(inflate_x, inflate_y) + + pygame.draw.ellipse(aura_surface, rgba, layer_rect) + + mid_rect = pygame.Rect(0, 0, spacing * 2, int(pillar_height * 0.35)) + mid_rect.center = (base_cx, cy) + bridge_color = base_colors[len(base_colors) // 2] + bridge_rgba = ( + bridge_color[0], + bridge_color[1], + bridge_color[2], + 130 + int(90 * self.motion_level), + ) + pygame.draw.ellipse(aura_surface, bridge_rgba, mid_rect) + + self.screen.blit(aura_surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 6: vertical ribbons + # ------------------------------------------------------------------ + def _pattern_vertical_ribbons(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + + num_ribbons = 10 + ribbon_width = self.width / num_ribbons + energy = self.motion_level + + for i in range(num_ribbons): + t = i / max(1, num_ribbons - 1) + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(210 * (0.7 + 0.7 * energy)) + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + + x_center = (i + 0.5) * ribbon_width + wobble = math.sin(self.time * 1.5 + i * 0.7) * ribbon_width * 0.25 * ( + 0.4 + 0.6 * energy + ) + rect = pygame.Rect( + 0, 0, + int(ribbon_width * 0.9), + int(self.height * (0.7 + 0.5 * self.render_scale)) + ) + rect.centerx = int(x_center + wobble) + rect.centery = int(self.height * 0.55) + + pygame.draw.rect(surface, rgba, rect, border_radius=40) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 7: grid pulse + # ------------------------------------------------------------------ + def _pattern_grid_pulse(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + + rows = 10 + cols = 16 + cell_w = self.width / cols + cell_h = self.height / rows + + energy = self.motion_level + + for r in range(rows): + for c in range(cols): + t_x = c / max(1, cols - 1) + t_y = r / max(1, rows - 1) + t = (t_x + t_y) / 2.0 + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + phase = self.time * 2.0 + c * 0.5 + r * 0.4 + pulse = (math.sin(phase) + 1.0) / 2.0 # 0~1 + + alpha = int(255 * pulse * (0.4 + 0.9 * energy)) + if alpha < 25: + continue + + size = int( + min(cell_w, cell_h) + * (0.26 + 0.55 * pulse) + * (0.8 + 0.8 * self.render_scale) + ) + x = int((c + 0.5) * cell_w) + y = int((r + 0.5) * cell_h) + + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + pygame.draw.circle(surface, rgba, (x, y), size) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 8: starfield + # ------------------------------------------------------------------ + def _pattern_starfield(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + + energy = self.motion_level + num_stars = int(260 * (0.9 + 0.9 * self.render_scale)) + + random.seed(42) + + for i in range(num_stars): + t = random.random() + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + x = random.randint(0, self.width) + y = random.randint(0, self.height) + + flicker = (math.sin(self.time * 3.0 + i * 0.21) + 1.0) / 2.0 + alpha = int( + 255 + * (0.22 + 0.88 * flicker) + * (0.4 + 1.0 * energy) + ) + alpha = max(0, min(alpha, 255)) + + size = 1 + int( + 3 * (0.35 + 0.65 * flicker) * (0.8 + 0.8 * self.render_scale) + ) + + rgba = (color[0], color[1], color[2], alpha) + pygame.draw.circle(surface, rgba, (x, y), size) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 9: vortex + # ------------------------------------------------------------------ + def _pattern_vortex(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + center_x, center_y = self._get_body_center() + + num_rings = 16 + max_radius = min(self.width, self.height) * 0.6 * (0.8 + 0.7 * self.render_scale) + + energy = self.motion_level + spin = self.time * (1.4 + 1.2 * energy) + + for i in range(num_rings): + t = i / (num_rings - 1) + radius = (t ** 1.1) * max_radius + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int( + 230 + * (1.0 - t ** 1.4) + * (0.6 + 0.9 * energy) + ) + alpha = max(0, min(alpha, 255)) + + angle_offset = spin + t * 3.0 + + num_segments = 64 + points = [] + for j in range(num_segments): + a = j / (num_segments - 1) * math.tau + angle_offset + x = center_x + math.cos(a) * radius + y = center_y + math.sin(a) * radius * 0.7 + points.append((int(x), int(y))) + + rgba = (color[0], color[1], color[2], alpha) + if len(points) > 1: + line_width = int(4 * self.stroke_boost) + pygame.draw.lines(surface, rgba, True, points, width=line_width) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 10: cross waves + # ------------------------------------------------------------------ + def _pattern_cross_waves(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + energy = self.motion_level + + # Horizontal bands + num_h = 8 + for i in range(num_h): + t = i / max(1, num_h - 1) + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + y = int( + (i + 0.5) * self.height / num_h + + math.sin(self.time * 1.5 + i) * 30 * (0.4 + 0.6 * energy) + ) + alpha = int(200 * (0.6 + 0.8 * energy)) + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + + half_h = int(20 * self.stroke_boost) + rect = pygame.Rect(0, y - half_h, self.width, half_h * 2) + pygame.draw.rect(surface, rgba, rect) + + # Vertical bands + num_v = 8 + for j in range(num_v): + t = j / max(1, num_v - 1) + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + x = int( + (j + 0.5) * self.width / num_v + + math.cos(self.time * 1.3 + j) * 30 * (0.4 + 0.6 * energy) + ) + alpha = int(170 * (0.6 + 0.8 * energy)) + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + + half_w = int(20 * self.stroke_boost) + rect = pygame.Rect(x - half_w, 0, half_w * 2, self.height) + pygame.draw.rect(surface, rgba, rect) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 11: aurora + # ------------------------------------------------------------------ + def _pattern_aurora(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + energy = self.motion_level + + bands = 4 + band_height = int(self.height * 0.28) + + for i in range(bands): + t = i / max(1, bands - 1) + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(210 * (0.5 + 0.6 * energy)) + rgba = (color[0], color[1], color[2], max(0, min(alpha, 255))) + + for x in range(0, self.width, 8): + phase = (x / self.width) * math.tau + self.time * 0.9 + i * 0.7 + y = int( + band_height * 0.2 + + (band_height * 0.4 + i * 30) * (0.4 + 0.6 * energy) + + math.sin(phase) * 35 * (0.4 + 0.8 * energy) + ) + rect = pygame.Rect(x, y, 12, band_height) + pygame.draw.rect(surface, rgba, rect) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 12: blooming orbs + # ------------------------------------------------------------------ + def _pattern_blooming_orbs(self, base_colors, dt): + center_x, center_y = self._get_body_center() + energy = self.motion_level + + # Spawn new orbs + target_count = int(50 * (0.5 + self.render_scale * 0.6)) + while len(self.bloom_orbs) < target_count: + angle = random.uniform(0, math.tau) + dist = random.uniform(30, min(self.width, self.height) * 0.4) + spawn_x = center_x + math.cos(angle) * dist + spawn_y = center_y + math.sin(angle) * dist * 0.8 + + max_radius = random.uniform(15, 45) * (0.7 + 0.7 * self.render_scale) + life = random.uniform(1.0, 3.0) + color_idx = random.randint(0, len(base_colors) - 1) + + self.bloom_orbs.append([spawn_x, spawn_y, 0.0, max_radius, life, color_idx]) + + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + new_list = [] + + for x, y, age, max_r, life, color_idx in self.bloom_orbs: + age += dt + if age > life: + continue + + t = age / life + radius = max_r * t + fade = (1.0 - t) ** 1.8 + + color = base_colors[color_idx] + alpha = int(255 * fade * (0.6 + 0.9 * energy)) + alpha = max(0, min(alpha, 255)) + + rgba = (color[0], color[1], color[2], alpha) + pygame.draw.circle(surface, rgba, (int(x), int(y)), int(radius)) + + new_list.append([x, y, age, max_r, life, color_idx]) + + self.bloom_orbs = new_list + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 13: comet trails + # ------------------------------------------------------------------ + def _pattern_comet_trails(self, base_colors, dt): + center_x, center_y = self._get_body_center() + energy = self.motion_level + + target_count = 25 + while len(self.comets) < target_count: + radius = random.uniform(120, min(self.width, self.height) * 0.5) + angle = random.uniform(0, math.tau) + speed = random.uniform(0.4, 1.2) + tail = random.randint(10, 25) + color_idx = random.randint(0, len(base_colors) - 1) + self.comets.append([radius, angle, speed, tail, color_idx]) + + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + new_list = [] + + for radius, angle, speed, tail, color_idx in self.comets: + angular_speed = speed * (1.0 + 1.3 * energy) + angle += angular_speed * dt + + color = base_colors[color_idx] + for i in range(tail): + t = i / max(1, tail - 1) + a = angle - t * 0.5 + r = radius * (0.7 + 0.3 * t) + + x = center_x + math.cos(a) * r + y = center_y + math.sin(a) * r * 0.8 + + alpha = int(255 * (1.0 - t) * (0.4 + 0.9 * energy)) + alpha = max(0, min(alpha, 255)) + size = int(8 * (1.0 - t) * (0.6 + 0.8 * self.render_scale)) + + rgba = (color[0], color[1], color[2], alpha) + pygame.draw.circle(surface, rgba, (int(x), int(y)), max(1, size)) + + new_list.append([radius, angle, speed, tail, color_idx]) + + self.comets = new_list + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + # PATTERN 14: spiral rings + # ------------------------------------------------------------------ + def _pattern_spiral_rings(self, base_colors): + surface = pygame.Surface((self.width, self.height), pygame.SRCALPHA) + center_x, center_y = self._get_body_center() + energy = self.motion_level + + num_layers = 8 + max_radius = min(self.width, self.height) * 0.55 * (0.7 + 0.7 * self.render_scale) + + for i in range(num_layers): + t = i / max(1, num_layers - 1) + radius = max_radius * (0.2 + 0.8 * t) + + if len(base_colors) == 1: + color = base_colors[0] + else: + idx = int(t * (len(base_colors) - 1)) + next_idx = min(idx + 1, len(base_colors) - 1) + local_t = (t * (len(base_colors) - 1)) - idx + color = self._lerp_color(base_colors[idx], base_colors[next_idx], local_t) + + alpha = int(230 * (0.4 + 0.8 * (1.0 - t)) * (0.6 + 0.7 * energy)) + alpha = max(0, min(alpha, 255)) + rgba = (color[0], color[1], color[2], alpha) + + segments = 32 + arc_len = math.tau * 0.6 + base_angle = self.time * (0.5 + 1.0 * energy) + i * 0.4 + + for j in range(segments): + a1 = base_angle + (j / segments) * arc_len + a2 = base_angle + ((j + 0.7) / segments) * arc_len + + x1 = center_x + math.cos(a1) * radius + y1 = center_y + math.sin(a1) * radius * 0.8 + x2 = center_x + math.cos(a2) * radius + y2 = center_y + math.sin(a2) * radius * 0.8 + + line_width = int(3 * self.stroke_boost) + pygame.draw.line(surface, rgba, (int(x1), int(y1)), (int(x2), int(y2)), line_width) + + self.screen.blit(surface, (0, 0)) + + # ------------------------------------------------------------------ + def _blit_camera(self, frame): + try: + h, w = frame.shape[:2] + except Exception: + return + + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frame_rgb = cv2.convertScaleAbs(frame_rgb, alpha=0.6, beta=0) + + scale = min(self.width / w, self.height / h) + new_size = (int(w * scale), int(h * scale)) + frame_resized = cv2.resize(frame_rgb, new_size) + + surf = pygame.surfarray.make_surface(frame_resized.swapaxes(0, 1)) + surf.set_alpha(150) + x = (self.width - new_size[0]) // 2 + y = (self.height - new_size[1]) // 2 + self.screen.blit(surf, (x, y)) + + + + # ------------------------------------------------------------------ + def _draw_label(self): + font = pygame.font.SysFont("arial", 26) + title = f"Energy Field: {self.spectrum_name}" + text = font.render(title, True, (245, 245, 245)) + self.screen.blit(text, (20, 18)) + + if self.current_profile: + elements_str = " · ".join(self.current_profile) + elif self.current_element: + elements_str = self.current_element + else: + elements_str = "None" + + sub_font = pygame.font.SysFont("arial", 22) + profile_text = sub_font.render( + f"Elements: {elements_str}", True, (230, 230, 230) + ) + self.screen.blit(profile_text, (20, 50)) + + # energy debug + debug_font = pygame.font.SysFont("arial", 16) + motion_text = debug_font.render( + f"Energy(cam): {self.motion_level:.2f}", True, (220, 220, 220) + ) + self.screen.blit(motion_text, (20, 80)) + + # ------------------------------------------------------------------ + def _lerp_color(self, c1, c2, t): + t = max(0.0, min(1.0, t)) + return ( + int(c1[0] + (c2[0] - c1[0]) * t), + int(c1[1] + (c2[1] - c1[1]) * t), + int(c1[2] + (c2[2] - c1[2]) * t), + ) + + def _boost_color(self, color, factor=2): + r, g, b = color + r = min(int(r * factor), 255) + g = min(int(g * factor), 255) + b = min(int(b * factor), 255) + return (r, g, b) + + # ------------------------------------------------------------------ + def reset_profile(self): + self.current_profile = None + self.current_element = None + self.spectrum_name = "None" + self.style = get_spectrum_style([]) # back to neutral style + self._clear_dynamic_buffers() + self.scale = 1.0 + self.temp_shift = 0.0 + print("[Animation] Profile cleared. Waiting for new selection.") + + def get_frame_surface(self): + surface = pygame.display.get_surface() + if surface is None: + return None + return pygame.surfarray.array3d(surface) + + + + + + + + diff --git a/Final Project/animation/set_profile.py b/Final Project/animation/set_profile.py new file mode 100644 index 0000000000..dc728d4d65 --- /dev/null +++ b/Final Project/animation/set_profile.py @@ -0,0 +1,388 @@ +from typing import List, Dict, Tuple + +Color = Tuple[int, int, int] +Style = Dict[str, object] + +# ---------------------------------------------------------------------- +# Base element color palette (primary hues) +# ---------------------------------------------------------------------- + +ELEMENT_MAIN_COLOR: Dict[str, Color] = { + "Fire": (255, 130, 70), + "Water": (80, 160, 255), + "Wind": (190, 230, 255), + "Earth": (110, 180, 110), + "Light": (255, 255, 230), + "Shadow": (150, 110, 200), +} + +ELEMENT_SECONDARY_COLOR: Dict[str, Color] = { + "Fire": (255, 210, 120), + "Water": (120, 220, 255), + "Wind": (150, 210, 255), + "Earth": (170, 220, 150), + "Light": (255, 245, 210), + "Shadow": (90, 60, 140), +} + +# Slightly different dark backgrounds to give each profile a distinct +# "night atmosphere" while still looking consistent. +BASE_BACKGROUNDS: Dict[str, Color] = { + "Fire": (10, 4, 8), + "Water": (3, 8, 16), + "Wind": (3, 10, 18), + "Earth": (5, 10, 6), + "Light": (8, 8, 18), + "Shadow": (6, 4, 10), +} + +# Global default background if nothing else is set +DEFAULT_BG: Color = (5, 7, 18) + + +def make_style( + name: str, + pattern_type: str, + base_colors: List[Color], + background_color: Color = DEFAULT_BG, + **kwargs, +) -> Style: + """Helper to create a style dict.""" + style: Style = { + "name": name, + "pattern_type": pattern_type, + "base_colors": base_colors, + "background_color": background_color, + } + style.update(kwargs) + return style + + +def element_pair_palette(elem: str) -> List[Color]: + """Return a simple 2-color palette for a single element.""" + c1 = ELEMENT_MAIN_COLOR.get(elem, (255, 255, 255)) + c2 = ELEMENT_SECONDARY_COLOR.get(elem, c1) + return [c1, c2] + + +def combo_palette(elems: List[str]) -> List[Color]: + """ + Return a 3-color palette for a 3-element combination. + The order of elems is already sorted by caller. + """ + colors: List[Color] = [] + for e in elems: + colors.append(ELEMENT_MAIN_COLOR.get(e, (255, 255, 255))) + return colors + + +# ---------------------------------------------------------------------- +# Single-element styles (fallback before 3-element profile is locked) +# ---------------------------------------------------------------------- + +SINGLE_ELEMENT_STYLES: Dict[str, Style] = { + "Fire": make_style( + name="Inner Flame", + pattern_type="pillar_orbs", + base_colors=element_pair_palette("Fire"), + background_color=BASE_BACKGROUNDS["Fire"], + pillar_width_factor=1.1, + halo_scale=1.3, + orb_count=40, + ), + "Water": make_style( + name="Tidal Heart", + pattern_type="ring_waves", + base_colors=element_pair_palette("Water"), + background_color=BASE_BACKGROUNDS["Water"], + ), + "Wind": make_style( + name="Sky Drift", + pattern_type="vortex", + base_colors=element_pair_palette("Wind"), + background_color=BASE_BACKGROUNDS["Wind"], + ), + "Earth": make_style( + name="Root Pulse", + pattern_type="double_pillar", + base_colors=element_pair_palette("Earth"), + background_color=BASE_BACKGROUNDS["Earth"], + ), + "Light": make_style( + name="Stellar Bloom", + pattern_type="starfield", + base_colors=element_pair_palette("Light"), + background_color=BASE_BACKGROUNDS["Light"], + ), + "Shadow": make_style( + name="Night Current", + pattern_type="galaxy", + base_colors=element_pair_palette("Shadow"), + background_color=BASE_BACKGROUNDS["Shadow"], + ), +} + +# ---------------------------------------------------------------------- +# 3-element combination styles (20 total) +# +# Key (always sorted): ("Earth", "Fire", "Light"), etc. +# ---------------------------------------------------------------------- + +COMBO_STYLES: Dict[Tuple[str, str, str], Style] = {} + +# 1. Earth, Fire, Light +COMBO_STYLES[("Earth", "Fire", "Light")] = make_style( + name="Solar Grove", + pattern_type="double_pillar", + base_colors=combo_palette(["Earth", "Fire", "Light"]), + background_color=(12, 6, 8), + pillar_width_factor=1.2, + pillar_height_factor=1.1, + halo_scale=1.4, +) + +# 2. Earth, Fire, Shadow +COMBO_STYLES[("Earth", "Fire", "Shadow")] = make_style( + name="Magma Veil", + pattern_type="vortex", + base_colors=combo_palette(["Fire", "Earth", "Shadow"]), + background_color=(10, 3, 10), +) + +# 3. Earth, Fire, Water +COMBO_STYLES[("Earth", "Fire", "Water")] = make_style( + name="Molten Delta", + pattern_type="cross_waves", + base_colors=combo_palette(["Fire", "Water", "Earth"]), + background_color=(6, 9, 12), +) + +# 4. Earth, Fire, Wind +COMBO_STYLES[("Earth", "Fire", "Wind")] = make_style( + name="Blazing Gale", + pattern_type="comet_trails", + base_colors=combo_palette(["Fire", "Wind", "Earth"]), + background_color=(10, 5, 8), +) + +# 5. Earth, Light, Shadow +COMBO_STYLES[("Earth", "Light", "Shadow")] = make_style( + name="Twilight Garden", + pattern_type="blooming_orbs", + base_colors=combo_palette(["Light", "Earth", "Shadow"]), + background_color=(6, 6, 12), +) + +# 6. Earth, Light, Water +COMBO_STYLES[("Earth", "Light", "Water")] = make_style( + name="Crystal Spring", + pattern_type="ring_waves", + base_colors=combo_palette(["Water", "Light", "Earth"]), + background_color=(4, 8, 14), +) + +# 7. Earth, Light, Wind +COMBO_STYLES[("Earth", "Light", "Wind")] = make_style( + name="Daybreak Breeze", + pattern_type="vertical_ribbons", + base_colors=combo_palette(["Wind", "Light", "Earth"]), + background_color=(6, 9, 16), +) + +# 8. Earth, Shadow, Water +COMBO_STYLES[("Earth", "Shadow", "Water")] = make_style( + name="Deep Moss", + pattern_type="grid_pulse", + base_colors=combo_palette(["Earth", "Water", "Shadow"]), + background_color=(4, 7, 9), +) + +# 9. Earth, Shadow, Wind +COMBO_STYLES[("Earth", "Shadow", "Wind")] = make_style( + name="Fogbound Roots", + pattern_type="aurora", + base_colors=combo_palette(["Shadow", "Wind", "Earth"]), + background_color=(5, 6, 10), +) + +# 10. Earth, Water, Wind +COMBO_STYLES[("Earth", "Water", "Wind")] = make_style( + name="Mist Valley", + pattern_type="spiral_rings", + base_colors=combo_palette(["Water", "Wind", "Earth"]), + background_color=(3, 8, 12), +) + +# 11. Fire, Light, Shadow +COMBO_STYLES[("Fire", "Light", "Shadow")] = make_style( + name="Lunar Flame", + pattern_type="galaxy", + base_colors=combo_palette(["Fire", "Light", "Shadow"]), + background_color=(10, 5, 12), +) + +# 12. Fire, Light, Water +COMBO_STYLES[("Fire", "Light", "Water")] = make_style( + name="Radiant Tide", + pattern_type="ring_waves", + base_colors=combo_palette(["Fire", "Water", "Light"]), + background_color=(5, 8, 13), +) + +# 13. Fire, Light, Wind +COMBO_STYLES[("Fire", "Light", "Wind")] = make_style( + name="Solar Storm", + pattern_type="radial_rays", + base_colors=combo_palette(["Fire", "Light", "Wind"]), + background_color=(12, 7, 10), +) + +# 14. Fire, Shadow, Water +COMBO_STYLES[("Fire", "Shadow", "Water")] = make_style( + name="Abyss Ember", + pattern_type="vortex", + base_colors=combo_palette(["Shadow", "Fire", "Water"]), + background_color=(6, 4, 10), +) + +# 15. Fire, Shadow, Wind +COMBO_STYLES[("Fire", "Shadow", "Wind")] = make_style( + name="Nocturne Blaze", + pattern_type="comet_trails", + base_colors=combo_palette(["Shadow", "Fire", "Wind"]), + background_color=(8, 5, 11), +) + +# 16. Fire, Water, Wind +COMBO_STYLES[("Fire", "Water", "Wind")] = make_style( + name="Storm Current", + pattern_type="pillar_orbs", + base_colors=combo_palette(["Fire", "Wind", "Water"]), + background_color=(5, 9, 14), + pillar_width_factor=0.9, + halo_scale=1.6, + orb_count=50, +) + +# 17. Light, Shadow, Water +COMBO_STYLES[("Light", "Shadow", "Water")] = make_style( + name="Moonlit Lake", + pattern_type="starfield", + base_colors=combo_palette(["Light", "Water", "Shadow"]), + background_color=(3, 7, 13), +) + +# 18. Light, Shadow, Wind +COMBO_STYLES[("Light", "Shadow", "Wind")] = make_style( + name="Veil of Dawn", + pattern_type="aurora", + base_colors=combo_palette(["Light", "Shadow", "Wind"]), + background_color=(6, 6, 12), +) + +# 19. Light, Water, Wind +COMBO_STYLES[("Light", "Water", "Wind")] = make_style( + name="Crystal Sky", + pattern_type="spiral_rings", + base_colors=combo_palette(["Light", "Wind", "Water"]), + background_color=(4, 9, 16), +) + +# 20. Shadow, Water, Wind +COMBO_STYLES[("Shadow", "Water", "Wind")] = make_style( + name="Midnight Surge", + pattern_type="blooming_orbs", + base_colors=combo_palette(["Shadow", "Water", "Wind"]), + background_color=(4, 5, 10), +) + + +# ---------------------------------------------------------------------- +# Default / fallback style +# ---------------------------------------------------------------------- + +DEFAULT_STYLE: Style = make_style( + name="Idle Constellation", + pattern_type="pillar_orbs", + base_colors=[(200, 200, 220), (150, 180, 220)], + background_color=DEFAULT_BG, + pillar_width_factor=1.0, + halo_scale=1.0, + orb_count=30, +) + + +# ---------------------------------------------------------------------- +# Public API +# ---------------------------------------------------------------------- + +def _normalize_profile(profile: List[str]) -> List[str]: + """Return a sorted list of unique element names.""" + unique = sorted(set(profile)) + return unique + + +def get_spectrum_style(profile: List[str] | None) -> Style: + """ + Map a list of elements (length 0–3) to a style dict. + + Cases: + - [] or None → neutral default style + - [E] → single-element style + - [E1, E2] → blended 2-element style (uses DEFAULT_STYLE-like pattern) + - [E1, E2, E3] → 3-element combination, one of 20 styles + """ + if profile is None: + profile = [] + + if len(profile) == 0: + return DEFAULT_STYLE + + profile = _normalize_profile(profile) + + # Single element + if len(profile) == 1: + elem = profile[0] + return SINGLE_ELEMENT_STYLES.get(elem, DEFAULT_STYLE) + + # Three elements → check combo table + if len(profile) >= 3: + key = tuple(profile[:3]) + style = COMBO_STYLES.get(key) + if style: + return style + # Fallback if not found for any reason: + return make_style( + name="Unknown Constellation", + pattern_type="pillar_orbs", + base_colors=combo_palette(list(key)), + background_color=DEFAULT_BG, + ) + + # Exactly 2 elements → simple blended style + if len(profile) == 2: + e1, e2 = profile + colors = [ + ELEMENT_MAIN_COLOR.get(e1, (255, 255, 255)), + ELEMENT_MAIN_COLOR.get(e2, (220, 220, 255)), + ] + return make_style( + name=f"{e1} · {e2} Field", + pattern_type="ring_waves", + base_colors=colors, + background_color=DEFAULT_BG, + ) + + # Should never reach here, but keep a safe fallback + return DEFAULT_STYLE + +def combo_palette(elems: List[str]) -> List[Color]: + colors: List[Color] = [] + for e in elems: + r, g, b = ELEMENT_MAIN_COLOR.get(e, (255, 255, 255)) + r = min(int(r * 2), 255) + g = min(int(g * 2), 255) + b = min(int(b * 2), 255) + colors.append((r, g, b)) + return colors + diff --git a/Final Project/images/16651762814138_.pic_hd.jpg b/Final Project/images/16651762814138_.pic_hd.jpg new file mode 100644 index 0000000000..913111c965 Binary files /dev/null and b/Final Project/images/16651762814138_.pic_hd.jpg differ diff --git a/Final Project/images/3D_printed.jpeg b/Final Project/images/3D_printed.jpeg new file mode 100644 index 0000000000..773d800644 Binary files /dev/null and b/Final Project/images/3D_printed.jpeg differ diff --git a/Final Project/images/Eneergy_setting.JPG b/Final Project/images/Eneergy_setting.JPG new file mode 100644 index 0000000000..e4415d4bc6 Binary files /dev/null and b/Final Project/images/Eneergy_setting.JPG differ diff --git a/Final Project/images/Energy_Card.JPG b/Final Project/images/Energy_Card.JPG new file mode 100644 index 0000000000..eb8de0963a Binary files /dev/null and b/Final Project/images/Energy_Card.JPG differ diff --git a/Final Project/images/Overall_Setup.jpeg b/Final Project/images/Overall_Setup.jpeg new file mode 100644 index 0000000000..6247550aa1 Binary files /dev/null and b/Final Project/images/Overall_Setup.jpeg differ diff --git a/Final Project/images/Scenario_1.jpg b/Final Project/images/Scenario_1.jpg new file mode 100644 index 0000000000..3c5ba725ce Binary files /dev/null and b/Final Project/images/Scenario_1.jpg differ diff --git a/Final Project/images/Scenario_2.jpg b/Final Project/images/Scenario_2.jpg new file mode 100644 index 0000000000..026e9e1d5c Binary files /dev/null and b/Final Project/images/Scenario_2.jpg differ diff --git a/Final Project/images/Scenario_3.jpg b/Final Project/images/Scenario_3.jpg new file mode 100644 index 0000000000..7754c3cfae Binary files /dev/null and b/Final Project/images/Scenario_3.jpg differ diff --git a/Final Project/images/Verplank_Diagram.jpg b/Final Project/images/Verplank_Diagram.jpg new file mode 100644 index 0000000000..9af8f01b81 Binary files /dev/null and b/Final Project/images/Verplank_Diagram.jpg differ diff --git a/Final Project/images/Wire_Setting.JPG b/Final Project/images/Wire_Setting.JPG new file mode 100644 index 0000000000..cbc6f28bdf Binary files /dev/null and b/Final Project/images/Wire_Setting.JPG differ diff --git a/Final Project/images/Wiring_Diagram.jpg b/Final Project/images/Wiring_Diagram.jpg new file mode 100644 index 0000000000..ff4c683945 Binary files /dev/null and b/Final Project/images/Wiring_Diagram.jpg differ diff --git a/Final Project/images/board_decoration.jpeg b/Final Project/images/board_decoration.jpeg new file mode 100644 index 0000000000..658ecf3320 Binary files /dev/null and b/Final Project/images/board_decoration.jpeg differ diff --git a/Final Project/images/elements.png b/Final Project/images/elements.png new file mode 100644 index 0000000000..8a9c356cff Binary files /dev/null and b/Final Project/images/elements.png differ diff --git a/Final Project/index.html b/Final Project/index.html new file mode 100644 index 0000000000..f6478efd2c --- /dev/null +++ b/Final Project/index.html @@ -0,0 +1,199 @@ + + + + +Inner Constellation + + + + + + + + + + + +
+

Inner Constellation

+

Energy Field Active

+ +
+
+ + + + + + + + diff --git a/Final Project/main.py b/Final Project/main.py new file mode 100644 index 0000000000..c6c4f05658 --- /dev/null +++ b/Final Project/main.py @@ -0,0 +1,73 @@ +import time +import pygame + +from sensors.sensor_manager import SensorManager +from animation.animation_engine import AnimationEngine + + +def main(): + pygame.init() + + sensors = SensorManager() + engine = AnimationEngine() + + print("System Started. Waiting for interactions...") + print("Press 'R' on the keyboard to re-select elements.") + print("Press 'ESC' or close the window to quit.") + + running = True + while running: + # ---------------------------- + # Handle global events + # ---------------------------- + for event in pygame.event.get(): + if event.type == pygame.QUIT: + running = False + + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_ESCAPE: + # ESC to quit + running = False + + elif event.key == pygame.K_r: + # R → reset profile selection + print("[Main] 'R' pressed to reset profile.") + sensors.reset_profile() + engine.reset_profile() + + # ---------------------------- + # Read sensors + # ---------------------------- + data = sensors.update() + + profile = data["profile"] # list or None + element = data["element"] # string + gesture = data["gesture"] # string or None + frame = data["frame"] # camera frame or None + + # ---------------------------- + # Update animation + # ---------------------------- + engine.update( + profile=profile, + element=element, + gesture=gesture, + proximity=None, # if you later add proximity, pass it here + frame=frame, + ) + + # Small sleep to avoid burning CPU + time.sleep(0.01) + + # Cleanup on exit + try: + if sensors.camera: + sensors.camera.release() + except Exception: + pass + + pygame.quit() + + +if __name__ == "__main__": + main() diff --git a/Final Project/prototype.png b/Final Project/prototype.png new file mode 100644 index 0000000000..913111c965 Binary files /dev/null and b/Final Project/prototype.png differ diff --git a/Final Project/readme.md b/Final Project/readme.md new file mode 100644 index 0000000000..f87f742a59 --- /dev/null +++ b/Final Project/readme.md @@ -0,0 +1,344 @@ + +# Final Project + +[Project Plan](#project-plan) + +[Functioning Project](#functioning-project) + +[Documentation of Design Process](#documentation-of-design-process) + +[Archive of All Code and Design Patterns](#archive-of-all-code-and-design-patterns) + +[Video Demo](#video-demo) + +[Reflections on Process](#reflections-on-process) + +[Group Work Distribution](#group-work-distribution) + +## Project Plan +Joy Sun(js888), Huiying Zhan (hz764), Hester Li(ql382) + +### Project Motivation +Technology has been changing our lives in a tremendous way. AI brought infinite capacity to our work and study setting, and relentless yet useless productivity is growing out of control. Promoting endless growth like that of cancer cells, capitalism refutes death, which means it also refutes life on the other hand. Human beings have been alienated into tools of labor, and their true attributes have been weakened. In a more dramatic way, our society is permeated with death drive. + +With the aim of bringing vitality back to our lives, our project serves as a trigger for people to reflect on their true forms of life. Six elements are symbols of the origins of the world, and the self-chosen element combinations refers to the person’s inner self, alter ego, in responding to the representatives of nature. This connection between inner power and visual patterns creates a dialogue between the self and the digital realm. Each interaction becomes a moment of connection between body and light, revealing an unseen aura that blurs the boundary between inner feeling and external form. + + +### Big Idea +Inner Constellation is an interactive art installation that visualizes a person’s inner energy as a living constellation made of light, color, and motion. It invites people to reflect on their emotions through simple, intuitive interactions. + +When a participant begins, they choose one of six symbolic elements — Fire 🔥, Water 💧, Wind ❄️, Earth 🌍, Light 💡, or Shadow 🧍‍♀️🧍‍♂️ — each representing a different emotional tone or personality energy. This choice is made by tapping an NFC card or selecting on screen. The system, powered by a Raspberry Pi, then generates a real-time animation that moves and transforms based on that element’s characteristics — for example, Fire glows and flickers, Water flows smoothly, Wind drifts and spins, Earth pulses steadily, Light radiates softly, and Shadow creates shifting patterns. + +A computer screen displays the animation blended with a live camera feed of the user, allowing them to see both their reflection and their personalized constellation at the same time. As they move their hands or adjust their posture, gesture and touch sensors detect these actions and feed them back into the system. The constellation reacts instantly — expanding, contracting, or changing colors — as if breathing together with the user. + +Through these visual and sensory responses, Inner Constellation transforms abstract emotions into tangible experiences. It connects technology and self-awareness in a poetic way, turning each participant’s reflection on the screen into a unique portrait of their energy and mood in that moment. +Inner Constellation concept image + +### Timeline +| **Milestone** | **Date** | **Notes / Details** | +|:--|:--:|:--| +| **Concept Lock & Story Development** | Nov 10–11 | Finalize core interaction and six energy elements. Write story concept and plan MVP scope. | +| **Visual MVP** | Nov 15 | Design an interactive energy circle that expands or contracts with hand gestures. | +| **Input Integration** | Nov 20 | Add gesture detection for circle control. Test responsiveness and stability. | +| **Projection Interaction** | Nov 22 | Project visualization on a large screen. Enable full-screen gesture control and adjust brightness. | +| **Parameter Tuning & Visual Polish** | Nov 25 | Refine color, motion, and gesture sensitivity. Add an “Energy Fortune” line and save-image feature. | +| **Functional Check-off** | Dec 1 | Demonstrate full flow: gesture → real-time motion → image save. Record performance notes. | +| **User Testing** | Dec 5 | Test with non-team users. Gather feedback on usability, aesthetics, and response. | +| **Final Presentation Prep** | Dec 6 | Prepare short demo video and slides. Present motivation, technical design, and evolution. | +| **Final Write-up & Repository Submission** | Dec 8 | Complete README, diagrams, and documentation. Upload final materials. | + +### Parts Needed +**The Device** +- 1× Raspberry Pi 4 Board +- 1× 32GB MicroSD Card w/ Card Reader +- 1× Computer Display / Monitor +- 1× USB Camera (for live reflection feed) +- 1× NFC Reader + NFC Cards (for element selection) +- 1× Gesture or Touch Sensor (e.g., APDS-9960 or Capacitive Pad) +- 1× HDMI Cable +- 1× USB Powered Speaker *(optional, for ambient sound)* +- 1× Power Supply for Raspberry Pi (5V 3A recommended) +- 1× Dupont Wire Set *(for sensor connections)* + +**For Exhibition Setup (optional)** +- 1× Projector *(for large-scale projection display)* +- 1× Tripod or Mounting Stand +- 1× External Light Diffuser or Frame *(for aesthetic setup)* + +### Fallback Plan + +If any hardware or sensor components fail, the system can still demonstrate the core experience through simplified input and display modes. + +- **Gesture Sensor Fails:** + Use keyboard or mouse input to manually trigger expansion/contraction of the energy circle. + +- **NFC Reader Malfunction:** + Replace NFC element selection with on-screen buttons for choosing Fire, Water, Wind, Earth, Light, or Shadow. + +- **Camera Not Working:** + Run the visualization without live reflection mode — display only the animated energy field on screen. + +- **Projector Unavailable:** + Switch to a standard monitor or laptop display for demonstration. + +- **Performance or Frame Rate Issues:** + Lower the resolution or particle density in the animation to maintain smooth real-time rendering. + +- **Sound Output Problem:** + Disable audio feedback and rely on visual responses only. + +*These fallback modes ensure the installation remains functional and visually expressive, even if some hardware components are unavailable.* + +## Functioning Project + +The following images show the fully functioning version of the **Inner Constellation** interaction system. +All hardware components—including the Raspberry Pi, MPR121 touch sensor, OLED display, USB camera, and six elemental touch cards—are wired together to create a seamless interactive experience. + +### 1. Wiring Setup (Live Hardware Layout) + +The first image shows the complete hardware arrangement during operation. +Each elemental card (Fire, Water, Wind, Earth, Light, Shadow) is connected to an MPR121 input via alligator clips. +Touching any card triggers a capacitive reading, which is then displayed on the OLED and sent to the animation engine. + +

+ +

+ +### 2. Element Selection Panel (Mounted Version) + +This second image shows the final mounted version of the element-selection board. +All six cards are neatly arranged in a 3D-printed frame, with copper foil pads at the bottom serving as capacitive touch surfaces. +The OLED display sits in the center to show real-time feedback of the selected element. + +

+ +

+ +Together, these components complete the functioning prototype, allowing users to physically select elements and influence the interactive constellation visualisation. + +### 3. Overall Setup (Final Exhibition Display) +The final installation brings all hardware, visuals, and interaction components together into a cohesive exhibition setup. +The projected animation reacts in real time to user-selected elements and their gestures, while the element panel sits nearby for intuitive touch-based input. +This setup was used during the final showcase, allowing visitors to explore their own “Inner Constellation” through physical interaction and responsive visual feedback. + +

+ +

+ + +## Documentation of Design Process +### Verplank Diagram + +![Verplank Diagram](images/Verplank_Diagram.jpg) + + +### Storyboards + +#### Scenario 1 +![Scenario 1](images/Scenario_1.jpg) + + +#### Scenario 2 +![Scenario 2](images/Scenario_2.jpg) + + +#### Scenario 3 +![Scenario 3](images/Scenario_3.jpg) + +### Wiring Diagram & Physical Setup +The wiring diagram below shows the full physical setup of our **Inner Constellation** prototype. A Raspberry Pi connects to the MPR121 capacitive touch breakout, an OLED display, a USB camera, and six element cards (Fire, Water, Wind, Earth, Light, Shadow). Each card is wired to one MPR121 input so that touching the copper pads on the cards selects an element, which is then visualized on the OLED and sent to the animation engine. + +Key components: + +1. **MPR121 capacitive touch sensor** – reads touch input from the six element cards. +2. **OLED display** – shows the currently selected element icon/state. +3. **USB camera** – detects user presence and movement for interaction. +4. **Element cards** – Fire, Water, Wind, Earth, Light, Shadow; each card is connected via alligator clips to the MPR121. +5. **Raspberry Pi** – runs the main loop, reads sensor data, and communicates with the visualisation on the main screen. + +![Wiring diagram for Inner Constellation prototype](images/Wiring_Diagram.jpg) + + +## Archive of All Code and Design Patterns +All related code are in: [animation](../Final%20Project/animation/) and [sensors](../Final%20Project/sensors/) + +### Sensor Layer Overview + +**Camera — Motion & Background Feed** + +The camera module continuously captures frames, computes motion energy by comparing consecutive grayscale images, and produces a softly blended background silhouette that contributes to the final animation. +Its interface is simple: `get_frame()` returns either a processed frame or `None` when unavailable. + +**OLED / TFT Display — Minimal Physical Feedback** + +The OLED display provides lightweight physical feedback by showing the currently selected element as well as the user's three-element profile. +If the device is not detected, the system automatically falls back to a dummy display mode to maintain pipeline stability. + +**MPR121 Touch Sensor — Element Selection** + +The MPR121 maps individual copper pads to six elemental identities—Fire, Water, Wind, Earth, Light, and Shadow. +Touch events are debounced for stability, and the input pipeline supports a three-step profile selection sequence used to generate a personalized spectrum. + +

+ +

+ + +### Animation Layer Overview + +**Overall Architecture** + +The animation engine renders at 60 FPS and integrates several input sources: time-based updates, motion-driven scaling, camera-derived features (motion level, body centroid, size estimation), and the user’s multi-element profile. +The system contains fourteen visual pattern modes, all implemented as modular pattern functions. + +**Core Logic** + +**1. Profile and Element System** +When a user completes the three-element sequence, the engine enters spectrum mode with blended palettes. +Single-element touches produce a fallback mode with a simplified color theme. +`get_spectrum_style()` returns base colors, background tones, preferred pattern type (such as galaxy, vortex, or pillar), and parameter presets such as orb speed or halo scale. + +**2. Camera-Derived Features** +The engine extracts motion intensity, approximate distance (size level), and a motion centroid representing horizontal and vertical body position. +These signals modulate animation behavior: motion affects breathing and expansion; horizontal position influences warmth vs. coolness in the color temperature; size level adjusts pillar width, orb radius, and other scale-sensitive effects. +A softly composited camera overlay (approximately 60% alpha) contributes to the ambient texture. + +**3. Energy Model** +A derived energy value governs parameters such as pillar height, halo radius, orb traversal speed, bloom strength, vortex depth, and grid brightness. +This single energy model allows different visual modes to respond consistently to user movement. + +**Visual Pattern System** +All patterns adapt automatically to the selected color spectrum, the user's motion energy, and the inferred distance from the camera. +This keeps visual output coherent across modes. + + +### Web Server Layer + +`server.py` coordinates two parallel systems: + +**Flask Web Server** +Serves the front-end interface (`index.html`), streams animation frames via MJPEG (`/frame`), and exposes simple control endpoints such as reset and UI toggles. + +**Pygame Animation Loop** +Runs in the main thread (required by SDL), receives continuous sensor updates, renders all animation frames, and shares the latest frame with Flask through thread-safe shared memory. + +Both processes remain synchronized through a `frame_lock`, ensuring stable frame delivery even under high interaction load. + + +### Tech Demo (Functional Checkoff) +This early demo shows the initial end-to-end functionality of our sensing pipeline. In this stage, the user selects an element by touching a specific position on the MPR121 sensor. The OLED screen displays the chosen element, and corresponding energy spots appear in the camera window to indicate the system’s recognition of that selection. + +**Tech Demo Video:** +[Watch the Demo](https://youtu.be/seF8rziH3oU) + +At this point, the system successfully connected touch input, OLED feedback, and camera-based visualization. Following this checkoff, we refined the energy spots and overall pattern animations—making them more dynamic, visually expressive, and directly responsive to gesture movements. We also implemented multi-element combinations to generate richer and more personalized energy patterns. These features were completed in the next development cycle and integrated into the final full-function version of the installation. + + + +## Video Demo +Below are three demo videos, each highlighting a different interaction feature: + +### 1. Motion-Responsive Pattern Movement +[Watch Video](https://youtu.be/BQ_63Se5rKE) +Shows how the visual patterns follow body movement with large motion amplitude. + +### 2. Color Temperature Shift (Left ↔ Right) +[Watch Video](https://youtu.be/R7gsMZxLD9I) +Demonstrates horizontal movement: right → warmer tones, left → cooler tones. + +### 3. Depth-Based Scaling (Forward ↔ Backward) +[Watch Video](https://youtu.be/6VBaitVTaA0) +Shows how stepping forward reduces pattern scale and stepping backward enlarges it. + + +## User Testing + +To understand how visitors interacted with *Inner Constellation*, we conducted several rounds of user testing in the actual exhibition environment. These tests helped us evaluate gesture responsiveness, distance sensing, element selection flow, and overall clarity of the experience. Through observing participants, we refined motion thresholds, improved the card selection feedback, and adjusted projection brightness to ensure patterns remained visible. + +Below are four user testing recordings, each capturing different aspects of real user interaction: + +- **[User Testing 1](https://youtu.be/hkju2jcZStg)** +- **[User Testing 2](https://youtu.be/3vFiwFm4fbo)** +- **[User Testing 3](https://youtu.be/uWHozAXnjfg)** +- **[User Testing 4](https://youtu.be/76HgVA5d4e8)** +From the tests, we learned several key insights: +- Users intuitively experimented with body movement, but needed clearer feedback about how distance influenced scale. +- Horizontal motion was well understood, especially once the color-temperature shift became visually distinct. +- Some users tried touching multiple cards at once, which helped us refine the touch debouncing and selection logic. +- The projection brightness and contrast needed tuning so that patterns remained visible even when users stood close to the screen. + +These findings helped us refine gesture sensitivity, adjust projection parameters, and streamline the element-selection flow before the final presentation. + + +## Reflections on Process +### Joy’s Reflection +Designing the visual identity of Inner Constellation became the core of my work. I spent countless hours experimenting with how color temperature, particle behavior, and motion density could shape emotion. Small visual shifts often changed the entire tone of an element. + +I built and tested many prototypes in Python, Pygame, Processing, and p5.py, studying how patterns responded to motion and blended in real time. Balancing visual richness with performance was tough, I often had to redesign animations to keep them fluid without losing depth. + +Collaborating with the sensing system also influenced every decision. The sensors weren’t just inputs, they defined how visuals moved and breathed with the user. Through that process, I realized how inseparable visual design and system behavior are in interactive work. + +This project taught me to see code as a creative medium, one that can express emotion as much as it executes logic. + + +### Hester’s Reflection +My work mainly focused on developing and stabilizing the sensing logic. Getting gesture signals, distance estimation, and touch inputs to behave reliably took a lot of tuning, otherwise the interaction felt noisy or accidental rather than intentional. Through repeated testing, I realized sensor-based interaction is shaped just as much by environment and user behavior as by code. Lighting, distance, and even how someone moves can completely change the system’s response. + +As I designed the interaction flow, I also started thinking about how to make the experience more playful and meaningful for the user. That’s when I began exploring the MPR121 touch-sensing feature: instead of asking people to click on a laptop, what if we created physical “element cards” they could touch? It felt more immersive and aligned better with the idea of an interactive ritual rather than a simple UI selection. + +I also replaced VNC with a web interface because the animation delay over VNC made the interaction feel disconnected. Running everything through a website made responses faster and created a smoother experience overall. + +This whole process taught me that interaction vocabularies don’t appear all at once, they evolve through trial, error, and constant adjustment. And the more I iterated, the more I learned to design not just for functional correctness, but for curiosity, playfulness, and the small moments that make an interface feel alive. + +### Sandy’s Reflection +My contribution focused on the parts of the project that connected the technical system to the final audience experience. In addition to preparing the README, demo video, user testing notes, and overall documentation, I took responsibility for much of the physical setup and exhibition logistics. I handled the full projector setup process—mounting and aligning the projector, calibrating projection size, adjusting brightness and contrast, and testing visibility in different ambient lighting conditions. I also arranged the physical interaction area, organized cable routing, positioned the Raspberry Pi and sensor board, and made sure the element cards were presented clearly to guide user behavior. + +I produced several physical components as well, including designing and assembling the 3D-printed card box and preparing the Energy Cards used during testing and exhibition. I tested the card layout, ensured the MPR121 connections stayed stable, and refined the spacing so users intuitively knew where to touch. + +User testing was another substantial part of my work. I filmed each interaction session and documented consistent patterns in how users approached the installation. These observations directly informed several adjustments, including clarifying the standing area, refining the projection distance for better visibility, and improving the way instructions were communicated to first-time participants. + +Across these responsibilities, my work bridged the technical system with the physical environment and the people interacting with it. I learned how much the success of an installation depends not only on code and visuals, but also on careful setup, environmental tuning, and clear communication with users. + +

+ +

+ +

+ +

+ +

+ +

+ + +## Group Work Distribution +### Joy +Joy focused on developing the interactive animation system and dynamic visual behavior of the installation. She implemented real-time energy field patterns that respond to user movement and camera input, designing and optimizing the animation engine with Python and Pygame. She collaborated on defining each elemental visual identity and ensured smooth, expressive transitions between energy states to maintain both aesthetic coherence and technical performance. + +**Deliverables:** +- `animation_engine.py` +- `set_profile.py` +- UI and Interaction Code (sensor and camera integration) +- Element Visual Style Sheets +- Final Exhibition Poster + + +### Hester +Hester was responsible for the interaction logic, sensing pipeline, and website build. Her work involved selecting and integrating the elemental sensors (including gesture, proximity, and touch-based inputs), designing how these signals translate into dynamic visual transitions, and ensuring that interactions felt intentional rather than noisy. She also built the web-based display system that replaced VNC, allowing the animation to run with lower latency and a smoother user experience. + +**Deliverables:** +- sensors design +- `server.py` +- Energy Element Cards (×6) + + +### Sandy +Sandy managed logistics, documentation, and user-facing presentation. She coordinated equipment (projector, backdrop, and decorative materials), prepared the mood board and final visual layout, conducted user testing (facilitation, observation, and interviews), and collected all required images and materials for the README. She also recorded and edited the final demo video and compiled the full project README. + +**Deliverables:** +- `README.md` +- User testing notes and documentation +- Demo video +- 3D printed card box +- Energy card design diff --git a/Final Project/requirements-pi.txt b/Final Project/requirements-pi.txt new file mode 100644 index 0000000000..e113511337 --- /dev/null +++ b/Final Project/requirements-pi.txt @@ -0,0 +1,21 @@ +# Core hardware control +adafruit-blinka +adafruit-circuitpython-busdevice +adafruit-circuitpython-mpr121 +adafruit-circuitpython-apds9960 +adafruit-circuitpython-rgb-display +adafruit-circuitpython-ssd1306 + +# Display & graphics +Pillow +pygame +flask + +# Camera +opencv-python-headless + +# Numeric / utils +numpy + +# Optional: debugging & MQTT communication +paho-mqtt diff --git a/Final Project/sensors/camera_reflection.py b/Final Project/sensors/camera_reflection.py new file mode 100644 index 0000000000..648c2b6728 --- /dev/null +++ b/Final Project/sensors/camera_reflection.py @@ -0,0 +1,30 @@ +import cv2 + +class CameraFeed: + def __init__(self, cam_index=0): + self.cap = cv2.VideoCapture(cam_index) + + if not self.cap.isOpened(): + print(f"[Camera] Failed to open camera at index {cam_index}. Running without camera.") + self.cap = None + return + + self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + + print("[Camera] Camera initialized successfully.") + + def get_frame(self): + if self.cap is None: + return None + + ret, frame = self.cap.read() + if not ret or frame is None: + return None + + frame = cv2.flip(frame, 1) + return frame + + def release(self): + if self.cap is not None: + self.cap.release() diff --git a/Final Project/sensors/gesture_apds9960.py b/Final Project/sensors/gesture_apds9960.py new file mode 100644 index 0000000000..1f2b1daa83 --- /dev/null +++ b/Final Project/sensors/gesture_apds9960.py @@ -0,0 +1,98 @@ +import board +import busio +from adafruit_apds9960.apds9960 import APDS9960 +import time + + +class GestureSensor: + """ + Wrapper around APDS9960 gesture engine. + + Mapping: + - Up → "expand" (grow energy field) + - Down → "shrink" (shrink energy field) + - Left → "cooler" (shift colors to cooler tones) + - Right → "warmer" (shift colors to warmer tones) + """ + + def __init__(self): + self.sensor = None + self.last_gesture_time = 0.0 + self.cooldown = 0.4 # seconds between two accepted gestures + + try: + i2c = busio.I2C(board.SCL, board.SDA) + self.sensor = APDS9960(i2c) + + # Proximity must be enabled for gesture engine + self.sensor.enable_proximity = True + + # Enable gesture mode + self.sensor.enable_gesture = True + + # Tunable parameters for better recognition + # Gain: higher → more sensitive + self.sensor.gesture_gain = 2 # 0=1x, 1=2x, 2=4x, 3=8x + + # These thresholds are somewhat empirical. + # If gestures are not detected, try lowering them a bit. + self.sensor.gesture_threshold_out = 10 + self.sensor.gesture_threshold_in = 30 + + # Optionally, you can also tweak gesture_dimensions + # and gesture_fifo_threshold here if needed. + + print("[Gesture] APDS9960 gesture engine initialized.") + + except Exception as e: + print(f"[Gesture] Failed to initialize APDS9960: {e}") + self.sensor = None + + def read_gesture(self): + """ + Poll the sensor and convert APDS9960 gesture codes into + high-level actions used by the animation engine. + + Returns: + "expand" / "shrink" / "cooler" / "warmer" / None + """ + if not self.sensor: + return None + + now = time.monotonic() + # Simple cooldown so one hand wave does not trigger many times + if now - self.last_gesture_time < self.cooldown: + # Still in cooldown → do not accept new gestures + return None + + # Sometimes reading once can miss a short gesture. + # We sample a few times very quickly. + decoded_action = None + for _ in range(3): + g = self.sensor.gesture() # Raw codes: 0x01 / 0x02 / 0x03 / 0x04 + if g == 0x01: + decoded_action = "expand" # Up + direction = "UP" + break + elif g == 0x02: + decoded_action = "shrink" # Down + direction = "DOWN" + break + elif g == 0x03: + decoded_action = "cooler" # Left + direction = "LEFT" + break + elif g == 0x04: + decoded_action = "warmer" # Right + direction = "RIGHT" + break + else: + # No valid gesture in this quick sample + continue + + if decoded_action is not None: + self.last_gesture_time = now + print(f"[Gesture] Detected {direction} to {decoded_action}") + return decoded_action + + return None diff --git a/Final Project/sensors/sensor_manager.py b/Final Project/sensors/sensor_manager.py new file mode 100644 index 0000000000..67f4159640 --- /dev/null +++ b/Final Project/sensors/sensor_manager.py @@ -0,0 +1,130 @@ +# sensors/sensor_manager.py + +from sensors.camera_reflection import CameraFeed +from sensors.tft_display import TFTDisplay +from sensors.touch_mpr121_selector import TouchElementSelector +# GestureSensor is intentionally not used (camera-only interaction) + + +class SensorManager: + def __init__(self): + print("[SensorManager] Initializing sensors...") + + # ------------------------- + # 1. OLED display + # ------------------------- + try: + self.tft = TFTDisplay() + print("[SensorManager] OLED/TFT ready (or dummy).") + except Exception as e: + print(f"[SensorManager] Failed to init TFTDisplay: {e}") + self.tft = None + + # ------------------------- + # 2. Touch selector (3 picks) + # ------------------------- + try: + self.touch_selector = TouchElementSelector(oled=self.tft) + print("[SensorManager] MPR121 touch selector ready.") + except Exception as e: + print(f"[SensorManager] Failed to init TouchElementSelector: {e}") + self.touch_selector = None + + # ------------------------- + # 3. Gesture sensor (DISABLED) + # ------------------------- + self.gesture = None + print("[SensorManager] Gesture sensor disabled (camera-only control).") + + # ------------------------- + # 4. Camera + # ------------------------- + try: + self.camera = CameraFeed() + print("[SensorManager] Camera feed ready (or disabled).") + except Exception as e: + print(f"[SensorManager] Failed to init CameraFeed: {e}") + self.camera = None + + # Runtime state + self.profile_selected = False # whether 3 elements have been locked in + self.user_profile = None # e.g. ["Fire", "Water", "Light"] + self.current_element = "None" # single element fallback before profile is locked + + # ---------------------------------------------------------------- + # MAIN UPDATE LOOP + # ---------------------------------------------------------------- + def update(self): + """ + Return a dict with: + profile: ["Water","Fire","Wind"] once the 3-element profile is locked + element: current single element name (fallback before profile selection) + gesture: always None (gesture sensor disabled) + frame: camera frame (or None) + """ + + # 1. Touch element selection (choose 3) + if not self.profile_selected and self.touch_selector is not None: + profile = self.touch_selector.update() + + if profile is not None: + # three elements chosen + self.profile_selected = True + self.user_profile = profile + print(f"[SensorManager] Final profile locked: {profile}") + + # show final on OLED + if self.tft: + self.tft.show_element_list(profile) + + # 2. Gesture sensor (disabled → always None) + gesture = None + + # 3. Camera frame + frame = None + if self.camera is not None: + try: + frame = self.camera.get_frame() + except Exception as e: + print(f"[SensorManager] Error reading camera: {e}") + frame = None + + # 4. Fallback element (only used before 3-pick is finished) + if not self.profile_selected: + if self.touch_selector and len(self.touch_selector.selected) > 0: + self.current_element = self.touch_selector.selected[-1] + else: + self.current_element = "None" + + # RETURN SENSOR DATA + return { + "profile": self.user_profile if self.profile_selected else None, + "element": self.current_element, + "gesture": gesture, # always None now + "frame": frame, + } + + # ---------------------------------------------------------------- + # RESET PROFILE (for 'R' key) + # ---------------------------------------------------------------- + def reset_profile(self): + """ + Clear current 3-element profile so the user can re-select elements. + + This is called from main.py when the user presses 'R'. + """ + # Reset internal state + self.profile_selected = False + self.user_profile = None + self.current_element = "None" + + # Reset touch selector (allow choosing 3 new elements) + if self.touch_selector is not None: + self.touch_selector.selected = [] + self.touch_selector.selection_done = False + + # Update OLED display if available + if self.tft is not None: + self.tft.show_element("Ready") + + print("[SensorManager] Profile reset. User can select new elements.") diff --git a/Final Project/sensors/tft_display.py b/Final Project/sensors/tft_display.py new file mode 100644 index 0000000000..80714636f1 --- /dev/null +++ b/Final Project/sensors/tft_display.py @@ -0,0 +1,91 @@ +try: + import board + import busio + from PIL import Image, ImageDraw, ImageFont + import adafruit_ssd1306 +except ImportError: + board = None + busio = None + Image = None + ImageDraw = None + ImageFont = None + adafruit_ssd1306 = None + + +class TFTDisplay: + def __init__(self): + self.display = None + self.enabled = False + + if not (board and busio and adafruit_ssd1306): + print("[OLED] Missing libraries, OLED disabled.") + return + + try: + i2c = board.I2C() + self.display = adafruit_ssd1306.SSD1306_I2C(128, 64, i2c, addr=0x3C) + self.display.fill(0) + self.display.show() + + # High-contrast monochrome buffer + self.image = Image.new("1", (128, 64)) + self.draw = ImageDraw.Draw(self.image) + + # Larger readable font + self.font = ImageFont.truetype( + "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 16 + ) + + self.enabled = True + print("[OLED] Ready (SSD1306 at 0x3C).") + + except Exception as e: + print(f"[OLED] init failed → {e}") + self.enabled = False + + # ------------------------------------------------------------ + # Show ONE element (big centered) + # ------------------------------------------------------------ + def show_element(self, name: str): + if not self.enabled: + print(f"[OLED] {name}") + return + + # Clear screen + self.draw.rectangle((0, 0, 128, 64), outline=0, fill=0) + + # Compute text size (use Pillow >=10 compatible API) + bbox = self.draw.textbbox((0, 0), name, font=self.font) + w = bbox[2] - bbox[0] + h = bbox[3] - bbox[1] + + # Center text + x = (128 - w) // 2 + y = (64 - h) // 2 + self.draw.text((x, y), name, font=self.font, fill=255) + + self.display.image(self.image) + self.display.show() + + # ------------------------------------------------------------ + # Show list of chosen elements (up to 3) + # ------------------------------------------------------------ + def show_element_list(self, elements): + if not self.enabled: + print("[OLED] Selected:", elements) + return + + # Clear screen + self.draw.rectangle((0, 0, 128, 64), outline=0, fill=0) + + # Title + self.draw.text((2, 2), "Chosen:", font=self.font, fill=255) + + # List elements + y = 20 + for e in elements: + self.draw.text((4, y), e, font=self.font, fill=255) + y += 16 + + self.display.image(self.image) + self.display.show() diff --git a/Final Project/sensors/touch_mpr121.py b/Final Project/sensors/touch_mpr121.py new file mode 100644 index 0000000000..897c95edbf --- /dev/null +++ b/Final Project/sensors/touch_mpr121.py @@ -0,0 +1,27 @@ + +import board +import busio +import adafruit_mpr121 + +# Element mapping: electrode → element +ELEMENT_MAP = { + 0: "Fire", + 1: "Water", + 2: "Wind", + 3: "Earth", + 4: "Light", + 5: "Shadow" +} + +class TouchSensor: + def __init__(self): + i2c = busio.I2C(board.SCL, board.SDA) + self.mpr121 = adafruit_mpr121.MPR121(i2c) + self.current_element = None + + def read_element(self): + for i in range(6): # only use 6 electrodes + if self.mpr121[i].value: + self.current_element = ELEMENT_MAP[i] + return self.current_element + return None diff --git a/Final Project/sensors/touch_mpr121_selector.py b/Final Project/sensors/touch_mpr121_selector.py new file mode 100644 index 0000000000..2fcb8bfdce --- /dev/null +++ b/Final Project/sensors/touch_mpr121_selector.py @@ -0,0 +1,63 @@ +import time +import board +import busio +import adafruit_mpr121 + + +class TouchElementSelector: + def __init__(self, oled=None): + i2c = board.I2C() + self.mpr121 = adafruit_mpr121.MPR121(i2c) + self.oled = oled + + self.element_map = { + 0: "Fire", + 1: "Water", + 2: "Wind", + 3: "Earth", + 4: "Light", + 5: "Shadow", + } + + self.selected = [] + self.selection_done = False + + print("[Touch] Element selector initialized.") + + def update(self): + """Read touch and return final 3-element profile once.""" + if self.selection_done: + return None + + for pad in range(6): + if self.mpr121[pad].value: + elem = self.element_map[pad] + + if elem not in self.selected: + self.selected.append(elem) + print(f"[Touch] Selected: {elem}") + + # Update OLED preview + if self.oled: + self.oled.show_element_list(self.selected) + + # Simple debounce + time.sleep(0.4) + + if len(self.selected) == 3: + self.selection_done = True + print(f"[Touch] Final user profile selected {self.selected}") + return self.selected + + return None + + def reset(self): + """Allow the user to select a new 3-element profile.""" + self.selected = [] + self.selection_done = False + + if self.oled: + # Clear OLED or show empty list + self.oled.show_element_list([]) + + print("[Touch] Selection reset. Ready for new profile.") diff --git a/Final Project/server.py b/Final Project/server.py new file mode 100644 index 0000000000..1d574ac686 --- /dev/null +++ b/Final Project/server.py @@ -0,0 +1,186 @@ + +import threading +import time +import os + +from flask import Flask, Response, send_from_directory +import numpy as np +import cv2 + +from animation.animation_engine import AnimationEngine +from sensors.sensor_manager import SensorManager + + +# --------------------------------------------------------- +# INITIALIZE GLOBAL OBJECTS +# --------------------------------------------------------- +app = Flask(__name__) + +engine = AnimationEngine() +sensors = SensorManager() + +latest_frame = None +frame_lock = threading.Lock() + +# NEW: web reset flag +reset_flag = False + +@app.route("/hide_labels", methods=["POST"]) +def hide_labels(): + """Hide text labels drawn by the animation engine (top-left text).""" + engine.show_labels = False + print("[Server] Labels hidden (show_labels = False).") + return "OK" + + +@app.route("/show_labels", methods=["POST"]) +def show_labels(): + """Show text labels drawn by the animation engine (top-left text).""" + engine.show_labels = True + print("[Server] Labels shown (show_labels = True).") + return "OK" + +print("System Started (Web Mode). Running Pygame in MAIN thread.") + + +# --------------------------------------------------------- +# HOME PAGE: serve index.html +# --------------------------------------------------------- +@app.route("/") +def index(): + """ + Serve the main Inner Constellation web page. + index.html must be in the same folder as server.py. + """ + base_dir = os.path.dirname(__file__) + return send_from_directory(base_dir, "index.html") + + +# --------------------------------------------------------- +# WEB RESET ENDPOINT +# --------------------------------------------------------- +@app.route("/reset", methods=["POST"]) +def reset_profile_web(): + """ + Web endpoint to request profile reset. + Called from the browser (Reset button). + """ + global reset_flag + reset_flag = True + print("[Server] Web reset requested via /reset") + return "OK" + + +# --------------------------------------------------------- +# MJPEG STREAM ENDPOINT +# --------------------------------------------------------- +@app.route("/frame") +def frame_feed(): + """Stream MJPEG frames from pygame render output.""" + + def generate(): + global latest_frame + while True: + with frame_lock: + frame = latest_frame.copy() if latest_frame is not None else None + + if frame is None: + time.sleep(0.03) + continue + + # Encode JPEG + ret, jpeg = cv2.imencode(".jpg", frame) + if not ret: + continue + + # Yield a multipart frame + yield ( + b"--frame\r\n" + b"Content-Type: image/jpeg\r\n\r\n" + + jpeg.tobytes() + + b"\r\n" + ) + + time.sleep(0.02) # ~50 FPS max + + return Response( + generate(), + mimetype="multipart/x-mixed-replace; boundary=frame" + ) + + +# --------------------------------------------------------- +# FLASK THREAD +# --------------------------------------------------------- +def start_flask(): + """Run Flask server in background.""" + app.run( + host="0.0.0.0", + port=8080, + debug=False, + threaded=True, + use_reloader=False, + ) + + +# --------------------------------------------------------- +# MAIN PYGAME LOOP (runs in main thread) +# --------------------------------------------------------- +def pygame_loop(): + global latest_frame, reset_flag + + while True: + # Sensor data + data = sensors.update() + + element = data.get("element") + gesture = data.get("gesture") + cam_frame = data.get("frame") + profile = data.get("profile") + proximity = data.get("proximity") # may be None + + # Update animation engine + engine.update( + profile=profile, + element=element, + gesture=gesture, + proximity=proximity, + frame=cam_frame, + ) + + # Handle reset from either: + # - R key inside pygame window (if you ever run with a real display) + # - Web /reset endpoint (reset_flag) + if getattr(engine, "request_reset", False) or reset_flag: + print("[Main] Reset requested (R key or web).") + sensors.reset_profile() + engine.reset_profile() + engine.request_reset = False + reset_flag = False + + # Convert pygame surface → numpy RGB → BGR + surf = engine.get_frame_surface() + if surf is not None: + try: + # surf shape: (width, height, 3) → transpose to (height, width, 3) + frame_rgb = np.transpose(surf, (1, 0, 2)) + frame_bgr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR) + + with frame_lock: + latest_frame = frame_bgr + except Exception as e: + print("[Server] Frame conversion error:", e) + + time.sleep(0.01) + + +# --------------------------------------------------------- +# ENTRY POINT +# --------------------------------------------------------- +if __name__ == "__main__": + # Start Flask in background thread + flask_thread = threading.Thread(target=start_flask, daemon=True) + flask_thread.start() + + # Run Pygame loop in main thread + pygame_loop() diff --git a/Lab 1/Fruit Costume.JPG b/Lab 1/Fruit Costume.JPG new file mode 100644 index 0000000000..85d4f6ca09 Binary files /dev/null and b/Lab 1/Fruit Costume.JPG differ diff --git a/Lab 1/Knife Costume.JPG b/Lab 1/Knife Costume.JPG new file mode 100644 index 0000000000..28ab987448 Binary files /dev/null and b/Lab 1/Knife Costume.JPG differ diff --git a/Lab 1/Overall_view.jpg b/Lab 1/Overall_view.jpg new file mode 100644 index 0000000000..73ef213854 Binary files /dev/null and b/Lab 1/Overall_view.jpg differ diff --git a/Lab 1/README.md b/Lab 1/README.md index cbc6dfa745..20a48b9b53 100644 --- a/Lab 1/README.md +++ b/Lab 1/README.md @@ -2,7 +2,7 @@ # Staging Interaction -\*\***NAME OF COLLABORATOR HERE**\*\* +**Jiayi Sun, Huiying Zhan, Qinrui Li** In the original stage production of Peter Pan, Tinker Bell was represented by a darting light created by a small handheld mirror off-stage, reflecting a little circle of light from a powerful lamp. Tinkerbell communicates her presence through this light to the other characters. See more info [here](https://en.wikipedia.org/wiki/Tinker_Bell). @@ -60,81 +60,138 @@ Labs are due on Mondays. Make sure this page is linked to on your main class hub ## Part A. Plan -To stage an interaction with your interactive device, think about: +\*\***Describe your setting, players, activity and goals here.**\*\* -_Setting:_ Where is this interaction happening? (e.g., a jungle, the kitchen) When is it happening? +- **Setting:** +A royal hall at night. The story has three short scenes: the maid with fruit, the queen with wine, and the general with a sword. Each scene reveals a motive and a plan to kill the King, with light effects used to build the mood and drive the drama. + +- **Players:** + - The King: the main target of all three plots. + - The Maid: a servant who wants revenge for her family. + - The Queen: once loved the King, now hates him after betrayal. + - The General: once a loyal soldier, now a rebel. + +- **Activity:** + 1. *The Maid’s Poisoned Fruit* — At the banquet, the maid brings a glowing fruit bowl. The King reaches for the fruit, the light turns red, but he pulls back. The glow fades. + 2. *The Queen’s Poisoned Wine* — The Queen raises a cup of glowing purple wine. She offers it to the King, his hand moves close, the light turns red, but he refuses to drink. The glow fades. + 3. *The Rebel’s Blade* — The general walks forward and takes his sword. The sword glows white, then red. He draws it, the King dies, and his crown falls. + +- **Goals:** + - Maid: to poison the King and take revenge. + - Queen: to kill the King with poisoned wine. + - General: to strike the King down with his sword. + - King: to survive these threats and keep his rule. -_Players:_ Who is involved in the interaction? Who else is there? If you reflect on the design of current day interactive devices like the Amazon Alexa, it’s clear they didn’t take into account people who had roommates, or the presence of children. Think through all the people who are in the setting. +\*\***Include pictures of your storyboards here**\*\* -_Activity:_ What is happening between the actors? +Here is the storyboard for our interaction design: -_Goals:_ What are the goals of each player? (e.g., jumping to a tree, opening the fridge). +![Storyboard](./storyboards.png) -The interactive device can be anything *except* a computer, a tablet computer or a smart phone, but the main way it interacts needs to be using light. +  -\*\***Describe your setting, players, activity and goals here.**\*\* -Storyboards are a tool for visually exploring a users interaction with a device. They are a fast and cheap method to understand user flow, and iterate on a design before attempting to build on it. Take some time to read through this explanation of [storyboarding in UX design](https://www.smashingmagazine.com/2017/10/storyboarding-ux-design/). Sketch seven storyboards of the interactions you are planning. **It does not need to be perfect**, but must get across the behavior of the interactive device and the other characters in the scene. +\*\***Summarize feedback you got here.**\*\* +Our peers thought the three-scene story was clear and engaging, with a strong progression from the maid to the queen to the general. They liked how each character’s motive for killing the King was distinct, and how the repeated moment of hesitation built suspense. A suggestion was to make the King’s role more expressive, showing more of his inner conflict, so that the audience can feel his tension as the threats escalate. -\*\***Include pictures of your storyboards here**\*\* +## Part B. Act out the Interaction -Present your ideas to the other people in your breakout room (or in small groups). You can just get feedback from one another or you can work together on the other parts of the lab. +\*\***Are there things that seemed better on paper than acted out?**\*\* +Yes. On paper, the glowing light effects (green, purple, white, red) looked very clear and dramatic. But when acting them out, it was harder to present the glow and control the timing of the color changes smoothly. Also, the emotional reactions of the King and the Queen felt stronger in the drawings than in the short performance, because it was hard to act them out. -\*\***Summarize feedback you got here.**\*\* +\*\***Are there new ideas that occur to you or your collaborator that come up from the acting?**\*\* +We realized that by adjusting the intensity of the lighting, we could highlight the interaction between the characters and the objects, and that adding background sound would make the video more vivid. +## Part C. Prototype the device -## Part B. Act out the Interaction +Code for the "Tinkerbelle" tool, and instructions for setting up the server and your phone are [here](https://github.com/IRL-CT/tinkerbelle). -Try physically acting out the interaction you planned. For now, you can just pretend the device is doing the things you’ve scripted for it. +\*\***Give us feedback on Tinkerbelle.**\*\* +This tool was quite easy to set up. Once we learned how to control the lighting, we use it flexibly. It was also very convenient, since it worked well with the phone and could be controlled from the computer. The phone screen changed colors instantly, which enhanced the performance effect. -\*\***Are there things that seemed better on paper than acted out?**\*\* +## Part D. Wizard the device +Take a little time to set up the wizarding set-up that allows for someone to remotely control the device while someone acts with it. Hint: You can use Zoom to record videos, and you can pin someone’s video feed if that is the scene which you want to record. -\*\***Are there new ideas that occur to you or your collaborator that come up from the acting?**\*\* +\*\***Include your first attempts at recording the set-up video here.**\*\* +In our first recording attempt, we prepared to manually control the lighting transitions. In the first two assassination scenes, where the King survives, we used a slow color shift in the lights to suggest rising tension before fading back to normal. In the final scene, to emphasize the success of the assassination, we highlighted the sword with a bright white glow followed by a rapid shift to red flashing light, underscoring the dramatic climax of the King’s death. +<<<<<<< HEAD +In the first try-out scene, where the Maid offers the poisoned fruit bowl to the King, the glow was controlled from the laptop and shifted in real time on the phone screen. At first, the bowl glowed green, but as the King reached out his hand, it turned red—showing that if he ate it, he would be poisoned. However, because of his suspicion, he pulled back, and the red glow faded away. -## Part C. Prototype the device +[Watch the set-up video on Youtube](https://www.youtube.com/watch?v=f_kJ1HM1AX8) -You will be using your smartphone as a stand-in for the device you are prototyping. You will use the browser of your smart phone to act as a “light” and use a remote control interface to remotely change the light on that device. -Code for the "Tinkerbelle" tool, and instructions for setting up the server and your phone are [here](https://github.com/IRL-CT/tinkerbelle). -We invented this tool for this lab! +Now, change the goal within the same setting, and update the interaction with the paper prototype. -If you run into technical issues with this tool, you can also use a light switch, dimmer, etc. that you can can manually or remotely control. +\*\***Show the follow-up work here.**\*\* +**Picture 1: The Stage** + -\*\***Give us feedback on Tinkerbelle.**\*\* +**Picture 2: The Maid and Fruit Bowl** + +**Picture 3: The Maid and Fruit Bowl on her Head** + -## Part D. Wizard the device -Take a little time to set up the wizarding set-up that allows for someone to remotely control the device while someone acts with it. Hint: You can use Zoom to record videos, and you can pin someone’s video feed if that is the scene which you want to record. +**Picture 4: The Queen and Wine Cup** + -\*\***Include your first attempts at recording the set-up video here.**\*\* +**Picture 5: The General and Sword** + -Now, change the goal within the same setting, and update the interaction with the paper prototype. -\*\***Show the follow-up work here.**\*\* +## Part E. Costume the device +Only now should you start worrying about what the device should look like. Develop three costumes so that you can use your phone as this device. -## Part E. Costume the device +\*\***Include sketches of what your devices might look like here.**\*\* +**Costume 1: Fruit Bowl** + + +**Costume 2: Wine Cup** + -Only now should you start worrying about what the device should look like. Develop three costumes so that you can use your phone as this device. +**Costume 3: Sword** + -Think about the setting of the device: is the environment a place where the device could overheat? Is water a danger? Does it need to have bright colors in an emergency setting? +\*\***What concerns or opportunitities are influencing the way you've designed the device to look?**\*\* +Our design comes from the storyline. We wanted to disguise the device as different weapons: a fruit bowl, a knife, and a cup of wine. We drew these objects, cut them out, and placed them over the phone screen. With Tinkerbelle’s lighting, the phone could show different effects—for example, green light for the fruit bowl, purple light for the wine, and white light for the knife. When each object turned into a weapon to kill the King, the light changed to red. -\*\***Include sketches of what your devices might look like here.**\*\* +### Photos of costumed devices +**Fruit Bowl** + -\*\***What concerns or opportunitities are influencing the way you've designed the device to look?**\*\* +**Wine** + +**Sword** + ## Part F. Record -\*\***Take a video of your prototyped interaction.**\*\* +\*\***Take a video of your prototyped interaction.**\*\* +[Watch the Prototyped Interaction video here](https://youtu.be/DuThEwBd8EE) -\*\***Please indicate who you collaborated with on this Lab.**\*\* -Be generous in acknowledging their contributions! And also recognizing any other influences (e.g. from YouTube, Github, Twitter) that informed your design. +\*\***Please indicate who you collaborated with on this Lab.**\*\* +I (Jiayi Sun) collaborated with Qinrui Li and Huiying Zhan. Huiying Zhan first proposed the storyline, while I took the lead in refining and writing the script. Qinrui and Huiying mainly worked on drawing the storyboards. All three of us contributed to making the props. During filming, I served as the main videographer and editor, Qinrui controlled the lighting, and Huiying arranged the actors’ movements and blocking. We worked together smoothly, and each of us made essential contributions to the group. The project would not have been complete without any one of us. We were very satisfied with the final outcome. We would also like to thank GitHub resources, the Tinkerbelle tool, and the iPhone recording software for their support. + +### Video of 3 prototyped interactions +[Watch the Scene 1 - The Maid’s Poisoned Fruit video here](https://youtu.be/yKgqXZjnT2M) +[Watch the Scene 2 - The Queen’s Poisoned Wine video here](https://www.youtube.com/watch?v=wbVWJKHsGkw) +[Watch the Scene 3 - The Rebel’s Drawn Blade video here](https://www.youtube.com/watch?v=w_AuT5ix660) +### Reflection +I have always been curious about directing and filmmaking, imagining one day creating short films, documentaries, or even running a personal media channel, but I had never truly tried it before. This project was the first real opportunity for me to step into that role. I worked on writing the script, designing the storyboard, and planning how each scene should be filmed. Through this process, I discovered how much I enjoy shaping a narrative and expressing ideas through both dialogue and visual structure. + +During the filming, I naturally took on the role of a director, planning how scenes should be staged, guiding the timing of actions, and later editing the footage into a silent-drama style piece. I found the editing especially rewarding, as it brought together all of our efforts into a coherent performance. Through this, I also began to understand more about cinematic techniques, such as how camera distance or angle can affect the tension of a scene, and how these choices influence the audience’s perception of the story. + +Finally, I feel that choosing to present our project in the form of a silent drama matched perfectly with the use of lighting as the central interactive element. The stark contrasts of light and shadow emphasized both the atmosphere of danger and the inner emotions of the characters. This lab not only helped me explore my long-standing interest in directing and editing, but also showed me how much I enjoy and am capable of this kind of creative work. + + +--- # Staging Interaction, Part 2 This describes the second week's work for this lab activity. @@ -144,7 +201,26 @@ This describes the second week's work for this lab activity. You will be assigned three partners from other groups. Go to their github pages, view their videos, and provide them with reactions, suggestions & feedback: explain to them what you saw happening in their video. Guess the scene and the goals of the character. Ask them about anything that wasn’t clear. -\*\***Summarize feedback from your partners here.**\*\* +\*\***Summarize feedback from your partners here.**\*\* +**Exploration of Design Space / Novelty of Concept** +- Settings were very creative; Storyboards were really well illustrated as well. +- I think it's a very interesting approach for this assignment, and it's very unique. However, they could have explored more ideas/settings in the storyboards since they only have one storyboard. I was a bit confused with the one storyboard, and I was not clearly sure if this is play or game. + +**Technical Execution** +- Designs were also really creative and had very nice execution, reflecting the light on paper props is a great idea. +- The video was very well made. The light transition was very clear! + +**Communication of Idea / Documentation of Process** +- Communication was good, but more explanation would be nice for why you chose medieval roles. +- I think the documentation was very neat; however, it would be more helpful if you could add a more general idea of what this is supposed to be! + +**Tested with User** +- They tested with acting out very well, and improved very well! + +**Overall** +- **Strengths:** Creativity, strong execution, artistic presentation, and innovative ideas. +- **Weaknesses:** Inconsistent, missing storyboards, ambiguous purpose of device functions, and sometimes insufficient user testing/feedback. + ## Make it your own @@ -154,3 +230,199 @@ Do last week’s assignment again, but this time: 3) We will be grading with an emphasis on creativity. \*\***Document everything here. (Particularly, we would like to see the storyboard and video, although photos of the prototype are also great.)**\*\* + +## Part A. Plan +\*\***Describe your setting, players, activity and goals here.**\*\* +- **Setting:** +The interaction takes place during a stage performance. The props are designed to enhance dramatic tension through light, sound, and vibration effects. + +- **Players:** +The prop master coordinates the devices. For example for the the story involves characters such as the king, maid, queen, and knight. Each character’s actions are emphasized through the props: lighting shifts, sound effects, and subtle vibrations create an immersive atmosphere for both actors and the audience. + +- **Activities:** +The props play an active role in advancing the dramatic storyline, particularly in the Lab1a scenes of the king’s assassination ((Detailed staging of these scenes can be seen in the lab1a video.) +). For example: + - In the first scene, the maid offers a poisoned fruit plate, with flashing lights that shift colors to draw attention and signal danger. + - In the second scene, the queen serves poisoned wine, and subtle vibrations make the glass tremble to reveal hidden menace. + - In the third scene, the knight strikes the king, accompanied by sharp sound effects that heighten the tension and impact of the action. + +Beyond this specific play, the same props can be adapted to different stage settings. A lighting designer could, for instance, create a wintry mood with alternating icy blue and white tones, or evoke autumn with dynamic transitions between yellow, orange, and red lights. These effects require close collaboration between different prop masters (lighting crew, sound designers, and stage decorators). + + +- **Goals:** +The goal of the prop team is to use these interactive effects to elevate the performance, making it more engaging, expressive, and memorable for the audience. + +\*\***Include pictures of your storyboards here**\*\* +**1. This storyboard demonstrates how the prop’s lighting effect can change the mood of a scene.** +- In Storyboards 1–3, the fruit plate is shown under different lighting: neutral, red, and orange. These variations signal shifting meanings, such as normal, danger, or tension. +- In Storyboards 4–6, the pumpkin is lit with neutral, dark, and warm orange light. The changes highlight different atmospheres, from calm to eerie to intense. + +![Storyboards 2](Storyboards%202.jpg) + +**2. This storyboard demonstrates how the prop’s sound effects can enhance dramatic expression on stage.** +- In Storyboards 1–2, a celebration scene is reinforced by the sound of clinking glasses, amplifying the joy and festivity of the moment. +- In Storyboards 3–4, an assassination scene is intensified by the sharp metallic sound of a sword, heightening the tension and danger between characters. +- In Storyboards 5–6, the classic fairytale moment is dramatized by the crisp crunch of an apple, emphasizing the fateful action and drawing the audience deeper into the story. + + + +\*\***Summarize feedback you got here.**\*\* +This storyboard clearly illustrates the scene and helps convey the narrative. The drawings are expressive and easy to follow, showing good effort in visual storytelling. However, the sequence could be expanded with more panels to better show transitions between actions, and adding notes on lighting or colors would make the intended atmosphere clearer. + + +## Part B. Act out the Interaction + +Try physically acting out the interaction you planned. For now, you can just pretend the device is doing the things you’ve scripted for it. + +\*\***Are there things that seemed better on paper than acted out?**\*\* +Yes. On paper the transitions between fruit plate colors and pumpkin lighting effects looked smooth and expressive, but when acted out it was harder to show the changes clearly without additional visual cues or narration. Some subtle differences in tone (red vs. orange light) were less obvious in practice than they appeared in the storyboard. + +\*\***Are there new ideas that occur to you or your collaborator that come up from the acting?**\*\* +Yes. While acting out the scenes, we realized that adding stronger contrasts in lighting or using sound effects could emphasize the changes more effectively. We also thought about combining multiple props (e.g., fruit plate and pumpkin together) in one scene to create a more dynamic narrative and highlight the color changes. + + +## Part C. Prototype the device + +You will be using your smartphone as a stand-in for the device you are prototyping. You will use the browser of your smart phone to act as a “light” and use a remote control interface to remotely change the light on that device. + +\*\***1. Auto Flash based on Tinkerbelle.**\*\* +The original Tinkerbelle only allowed **manual** color switching via a color picker. We extended it with an **Auto Flash** system and several reliability/UX tweaks. + +**New features** +- **Auto Flash (automatic color cycling):** screen color changes automatically without manual clicks. +- **Two animation modes:** `blink` (instant switch) and `fade` (smooth transition). +- **Configurable sequence:** comma-separated color list (e.g., `#ff0000,#000000,#ff6f61`). +- **Adjustable speed:** `Step(ms)` to control the dwell time between colors. +- **Loop toggle:** repeat the sequence or run once and stop. +- **Start/Stop controls:** UI buttons to launch or halt the auto sequence at any time. +- **Broadcast to all screens:** controller actions can propagate to every connected “light” screen. + +**Server (Flask + Socket.IO) changes** — *`app.py`* +- Added new events and broadcast helpers: + - `autoFlash` → `broadcast('autoFlash', cfg)` + - `stopAuto` → `broadcast('stopAuto', {})` +- Kept existing events (`hex`, `audio`, `pauseAudio`) and standardized broadcasting through a single `broadcast()` helper. + +**Client JS changes** — *`static/index.js`* +- Emits `autoFlash` with `{ colors, stepMs, mode, loop, ease }`. +- Listens for `autoFlash` and runs an interval to step through colors. +- Supports **fade** by applying `background-color` CSS transitions, or **blink** with no transition. +- Adds `stopAuto()` to clear timers and reset state when `stopAuto` is received. +- Minor UX: reveal/hide the controller panel, destroy/recreate Pickr when switching modes, enable audio playback on Safari (mute-unmute trick). + +**HTML / UI changes** — *`templates/index.html`* +- Added an **Auto Flash panel** with: + - Colors input + - Step (ms) input + - Mode selector (`blink`/`fade`) + - Loop checkbox + - **Start Auto** / **Stop** buttons +- Kept the original **Jane Wren** (controller) and **Tinkerbelle** (light) roles. + +**How to use** +1. Open the page on the **controller** device, click **Jane Wren** to enter control mode. +2. In **Auto Flash**, set: colors, step (ms), mode, and loop → click **Start Auto**. +3. On **light** devices, click **Tinkerbelle** (fullscreen). All connected lights will follow the broadcasted auto colors. +4. Click **Stop** to halt the sequence or send a new configuration. + +**Why this improves the original** +- Enables **hands-free** lighting cues for stage/testing. +- Provides **reproducible** timing and transitions (good for demos and user tests). +- Scales to **multiple displays** via broadcasts, keeping scenes synchronized. + +\*\***2. Sound Effects based on Tinkerbelle.**\*\* +We also added 6 different sounds effect function to the tinkerbelle, enabling the device to better capture and express the emtions of the actors in stage settings. + +**Single Global Audio System** + - Uses one shared `Audio` object for all playback, avoiding overlap and conflicts. +**Controller ↔ Light Synchronization** + - Controller sends `audio` / `pauseAudio` events via Socket. + - Light devices (e.g., mobile phones) play or stop sounds accordingly. +**Custom Input Playback** + - Type a sound name in the input box and press **Play** → Controller emits it, Light plays it. +**Quick Sound Buttons** + - Pre-bound to common effects (e.g., `thunder`, `laugh`, `eatfull`, `drum`, `explosion`, `clap`). + - One click = instant sound effect across all Light clients. + - Button highlights briefly (green flash) for feedback. +**Local Sound Library** + - Sounds stored under `/static/sounds/` (e.g., `static/sounds/thunder.mp3`). + - Easy to extend — just add new `.mp3` files and bind them with `bindSoundButton(id, filename)`. + +\*\***3. Screen Shake based on Tinkerbelle.**\*\* +Finally, we added a screen shake effect to imitate the viberation of the tool, which finally focus on the sound effect of a wine cup due to the restriction of ios system. + +**Generic Shake** + - `triggerShake(duration)` applies a brief vibration effect on the screen to add visual impact. +**Sound-linked Shake** + - Small shakes are triggered when sounds are played. + - The `wineBtn` not only plays audio but also creates a **5-second continuous shake**. +**Gyroscope Integration** + - On mobile devices, orientation sensors enhance interactivity: + - Tilting the device adjusts screen brightness. + - Exceeding a tilt threshold triggers a shake feedback. + + +\*\***4. Limitations & Decisions**\*\* +**Mobile vibration (iOS restriction)** +We originally planned to implement continuous vibration on mobile devices. However, due to system restrictions on iOS, true vibration is not possible. + +**Workaround: screen shaking** +As a substitute, we implemented on-screen shaking effects. During testing, we found the effect was not very noticeable, especially when phones were wrapped inside stage props. + +**Final decision: focus on sound** +Given the limitations, we decided to rely on sound and color effects as the primary enhancements. Sounds and Lights provided clearer feedback and a stronger immersive impact for both actors and the audience. +Even though vibration cannot currently be realized through code on iOS devices, we believe it remains a valuable interaction. In our product vision, vibration feedback should be retained as a design feature, to be enabled once technical or hardware conditions allow. + + +## Part D. Wizard the device +Take a little time to set up the wizarding set-up that allows for someone to remotely control the device while someone acts with it. Hint: You can use Zoom to record videos, and you can pin someone’s video feed if that is the scene which you want to record. + +\*\***Include your first attempts at recording the set-up video here.**\*\* +[Watch the setup video for Auto Flash lighting controller here](https://youtu.be/pGlznBV1fC4) + +[Watch the setup video for Viberation controller here](https://youtu.be/OM1hdqViYhQ) + +[Watch the setup video for Sounds Effect controller here](https://youtu.be/TQONw32uM00) + +## Part E. Costume the device +\*\***Include sketches of what your devices might look like here.**\*\* +This prop is designed as a simple stage device that combines **lighting, sound, and vibration** to support theatrical performance. + +- **Lighting Effect:** A small spotlight mounted on the top projects light. It can shift colors to match different moods and scenes. +- **Sound Effect:** Built-in speakers on the sides of the device produce immersive audio, such as sound effects or ambient cues. +- **Detachable Vibration Effect:** The base is equipped with a vibration module that makes the device shake slightly. The unit is detachable and can also be mounted onto different stage props to create vibration effects as needed. + + +\*\***What concerns or opportunitities are influencing the way you've designed the device to look?**\*\* +The design is shaped by both practical concerns and opportunities for stage use. We needed the prop to clearly show its three functions—lighting, sound, and vibration—so the appearance emphasizes these features with a visible lamp, speaker units, and a detachable vibration module. The opportunity here is to make the device simple and flexible: the vibration module can be removed and attached to other props, giving more creative options for stage effects. The minimal, box-like design ensures that the device looks neutral and can blend into different theater settings while still being easy to operate during performances. + +## Part F. Record + +\*\***Take a video of your prototyped interaction.**\*\* +- This following video demonstrates the prototyped interaction for our Lab1b lighting function. The device is used as a stage lighting prop that automatically switches between selected colors at fixed time intervals. In the video, warm tones such as red, orange, and yellow are applied to a fruit plate prop. The gradual color transitions highlight the fruits, making them appear more vivid and easier for the audience to notice during a performance. This showcases how the lighting effect can enhance the visibility and expressiveness of different stage props in theatrical settings. + +[Watch the prototyped interaction video for lighting effect here](https://www.youtube.com/watch?v=gxWBAg7dB6A) + + +- This following video demonstrates the prototyped interaction for our Lab1b sound effects function. The device is used as a stage prop controller that triggers different sound effects in real time during a performance. In the video, six distinct sounds — thunder, laughter, eating, drum, explosion, and applause — are assigned to buttons on the controller. When pressed, the corresponding sound is immediately played on the stage prop device (a smartphone). +These effects are applied to dramatize key moments of a theatrical performance: thunder emphasizes tension in a storm scene, laughter highlights irony in dialogue, eating underscores the banquet setting, drums enhance dramatic build-ups, explosions mark climactic action, and applause provides closure to a scene. By integrating these sound effects into stage action, the prototype showcases how interactive technology can enrich the audience’s sensory experience and amplify the emotional impact of live theatre. + +[Watch the prototyped interaction video for sounds effect here](https://youtu.be/xM0MV5j1-60?si=PocSgRw18hESC5kb) + +- Now please watch this video for fun!! + +[Watch for fun here](https://youtu.be/G2aA-YXLc8Y?si=kg5IEFqaYFNeC0AA) + + +\*\***Please indicate who you collaborated with on this Lab.**\*\* +I collaborated with Qinrui Li and Huiying Zhan on this Lab. I mainly focused on the coding and design of the sounds effect function. Qinrui was responsible for the vibration effect, while Huiying (Sandy) worked on the auto flash lighting function. Our collaboration went very smoothly, as each of us contributed a unique feature that complemented the others. Together, the lighting, vibration, and sound effects enriched the overall stage atmosphere and made the theatrical presentation more immersive. + +We would like to thank the YouTube platform for providing a convenient channel to upload and share our video of the prototyped interaction. We also acknowledge the Tinkerbelle coding prototype as an inspiration and foundation for our different effects' design. In addition, the Jianying app was very helpful for video editing, enabling us to present our prototype clearly and effectively. + +## Reflection +Developing the sound effects function was one of the most challenging parts of our project. I went through many iterations — starting from trying to fetch sounds online, then downloading them locally, arranging them into a foldable button panel, and finally refining the layout into a clean grid. Each step required rethinking the code structure, and the debugging process was not easy. +Despite the challenges, the final sound effects function works well and contributes significantly to the theatrical prototype. It effectively captures sudden shifts in mood and adds expressive layers to the stage performance. This showed me how interactive audio can complement lighting to enhance storytelling. +Lastly, this work was also a great team effort. While I focused on coding and iteration of the sound effects, my teammates supported with viberation and auto flash lighting. Together, we were able to push the prototype forward and make it both functional and expressive. + + + diff --git a/Lab 1/Storyboards 2.jpg b/Lab 1/Storyboards 2.jpg new file mode 100644 index 0000000000..11e40b76a2 Binary files /dev/null and b/Lab 1/Storyboards 2.jpg differ diff --git a/Lab 1/The_General_and_Sword.jpg b/Lab 1/The_General_and_Sword.jpg new file mode 100644 index 0000000000..813237e796 Binary files /dev/null and b/Lab 1/The_General_and_Sword.jpg differ diff --git a/Lab 1/The_Maid_and_Fruit_Bowl.jpg b/Lab 1/The_Maid_and_Fruit_Bowl.jpg new file mode 100644 index 0000000000..c6dac04f5d Binary files /dev/null and b/Lab 1/The_Maid_and_Fruit_Bowl.jpg differ diff --git a/Lab 1/The_Maid_and_Fruit_Bowl_2.jpg b/Lab 1/The_Maid_and_Fruit_Bowl_2.jpg new file mode 100644 index 0000000000..4444975e8a Binary files /dev/null and b/Lab 1/The_Maid_and_Fruit_Bowl_2.jpg differ diff --git a/Lab 1/The_Queen_and_Wine.jpg b/Lab 1/The_Queen_and_Wine.jpg new file mode 100644 index 0000000000..7e4f6625d6 Binary files /dev/null and b/Lab 1/The_Queen_and_Wine.jpg differ diff --git a/Lab 1/Wine Costume.jpg b/Lab 1/Wine Costume.jpg new file mode 100644 index 0000000000..0a7e82d962 Binary files /dev/null and b/Lab 1/Wine Costume.jpg differ diff --git a/Lab 1/prop.png b/Lab 1/prop.png new file mode 100644 index 0000000000..6df1c60483 Binary files /dev/null and b/Lab 1/prop.png differ diff --git a/Lab 1/storyboards 3.png b/Lab 1/storyboards 3.png new file mode 100644 index 0000000000..4b4ff27ba4 Binary files /dev/null and b/Lab 1/storyboards 3.png differ diff --git a/Lab 1/storyboards.png b/Lab 1/storyboards.png new file mode 100644 index 0000000000..e4ea9f933a Binary files /dev/null and b/Lab 1/storyboards.png differ diff --git a/Lab 2/README.md b/Lab 2/README.md index fdf299cbbf..09954ace81 100644 --- a/Lab 2/README.md +++ b/Lab 2/README.md @@ -1,5 +1,5 @@ # Interactive Prototyping: The Clock of Pi -**NAMES OF COLLABORATORS HERE** +**Jiayi Sun and Huiying Zhan** Does it feel like time is moving strangely during this semester? @@ -152,11 +152,19 @@ You can type the name of a color then press either of the buttons on the MiniPiT ``` #### Displaying Info with Texts -You can look in `screen_boot_script.py` for how to display text on the screen! +You can look in `screen_boot_script.py` for how to display text on the screen! + #### Displaying an image -You can look in `image.py` for an example of how to display an image on the screen. Can you make it switch to another image when you push one of the buttons? +You can look in `image.py` for an example of how to display an image on the screen. Can you make it switch to another image when you push one of the buttons? + +See Interactions Here + + + +- [▶️ Watch video for colors interaction here](https://youtu.be/n4wuPcWpbyE) +- [▶️ Watch video for images interaction here](https://youtu.be/JL0R7H22HcI) @@ -182,14 +190,47 @@ Now you should be able to edit python scripts with Thonny on your Pi. Option 3. A nowadays often preferred method is to use Microsoft [VS code to remote connect to the Pi](https://www.raspberrypi.com/news/coding-on-raspberry-pi-remotely-with-visual-studio-code/). This gives you access to a fullly equipped and responsive code editor with terminal and file browser. -Pro Tip: Using tools like [code-server](https://coder.com/docs/code-server/latest) you can even setup a VS Code coding environment hosted on your raspberry pi and code through a web browser on your tablet or smartphone! +Pro Tip: Using tools like [code-server](https://coder.com/docs/code-server/latest) you can even setup a VS Code coding environment hosted on your raspberry pi and code through a web browser on your tablet or smartphone! +See Clock Here + + ## Part E. Now moved to Lab2 Part 2. ## Part F. Now moved to Lab2 Part 2. -## Part G. -## Sketch and brainstorm further interactions and features you would like for your clock for Part 2. +## Part G +## Sketch and brainstorm further interactions and features you would like for your clock for Part 2 + +--- + +### Picture 1: Roosevelt Island Cable Clock + + +- **Concept**: A creative clock interface that uses the number of Roosevelt Island cable car round trips to represent time. +- **Composition**: The interface consists of a single horizontal blue cable with a blue cable car hanging from it. Inside the cable car, a number shows the trip count, representing the passage of time. +- **Unit of Time**: Each round trip equals 30 minutes. The current time in a day is calculated by converting hours into minutes, dividing by 30, and rounding down. +- **How it Works**: + - At the top, the text reads *“CABLE CAR ROUND TRIPS”*, which is the unit of time in our design. + - The cable car moves gradually from the bottom-left corner to the top-right corner over the course of the day. + - The background shifts from light red to dark red, simulating time passing. + - The entire design uses pixel art style, making it playful and electronic. + +--- + +### Picture 2: Interactive Reminder Clock + + +- **Concept**: A pixel art watch interface that combines current time with reminder notifications. +- **Composition**: A horizontal digital watch screen. The current time (HH:MM) is displayed at the top in large pixelated digits, while below it a reminder text appears, such as *“Reminder: Meeting.”* +- **Unit of Time**: The reminder is measured with a countdown that always fits within one hour. For example, the display may show *“45 min remaining”*, indicating how much time is left until the event begins. +- **How it Works**: + - Users type in upcoming events on their own. + - Displays both the present time and the urgency of upcoming tasks. + - By pairing the clock with a real-time countdown, the user sees not only the time but also how soon the next activity will start. + - The retro pixel art style reinforces a clear, playful, and distinctive identity. + +--- # Prep for Part 2 @@ -208,18 +249,48 @@ Does time have to be linear? How do you measure a year? [In daylights? In midni Can you make time interactive? You can look in `screen_test.py` for examples for how to use the buttons. + Please sketch/diagram your clock idea. (Try using a [Verplank diagram](https://ccrma.stanford.edu/courses/250a-fall-2004/IDSketchbok.pdf))! +**CABLE CAR ROUND TRIPS** + + +## Key Features + +- **Dynamic gradient background** – background colors shift over time, making the passage of time more vivid. +- **Cable car animation** – the car moves smoothly along the cable, with a **gentle pendulum swing**, simulating natural motion. +- **Stylized design** – car body is a **rounded rectangle with a trapezoid roof**, sky-blue color, and a subtle shadow effect. +- **Center-aligned text** – the trip count is displayed clearly in the middle of the car. +- **Explanatory text** – note at the bottom reminds users that one round trip = 30 minutes. + + + **Reminder Clock** + + + + **We strongly discourage and will reject the results of literal digital or analog clock display.** -\*\*\***A copy of your code should be in your Lab 2 Github repo.**\*\*\* +\*\*\***A copy of your code should be in your Lab 2 Github repo.**\*\*\* + A copy of my code is included in my Lab 2 GitHub repo: + +**gesture_daemon.py** – Implements gesture control. When the user waves their hand toward the screen, the detailed reminder view toggles on; waving again hides the reminder so that only the time is shown. +**reminder_clock.py** – Implements button interactions. Button A cycles to the next reminder, Button B cycles to the previous reminder, and pressing A + B together deletes the current reminder. + +**screen_cable_clock.py** – Implements the animated cable car clock with dynamic gradient background and pendulum-style swinging motion. + +--- ## Assignment that was formerly Part F. ## Make a short video of your modified barebones PiClock -\*\*\***Take a video of your PiClock.**\*\*\* +\*\*\***Take a video of your PiClock.**\*\*\* + **Here is the video of my Reminder Clock demo: 👉 [https://youtu.be/Pi32cqu7j2A](https://youtu.be/Pi32cqu7j2A)** + + **Here is the video of my Cable Clock demo: 👉 [https://youtu.be/d-vsA0yDSq8](https://youtu.be/d-vsA0yDSq8)** + After you edit and work on the scripts for Lab 2, the files should be upload back to your own GitHub repo! You can push to your personal github repo by adding the files here, commiting and pushing. @@ -240,4 +311,14 @@ As always, make sure you document contributions and ideas from others explicitly You are permitted (but not required) to work in groups and share a turn in; you are expected to make equal contribution on any group work you do, and N people's group project should look like N times the work of a single person's lab. What each person did should be explicitly documented. Make sure the page for the group turn in is linked to your Interactive Lab Hub page. +## Reflection + +Throughout this project, I went through multiple **iterations**: + +- Initially, I only had a static black background and a moving car, but no real personality. +- Then I experimented with **dynamic gradient backgrounds** to make the time passage more vivid. +- I refined the car’s **appearance** (adding rounded corners, a trapezoid roof, windows, and shadow effects) so that it looks more polished. +- I also made sure the **text alignment** was centered properly inside the car. +- Finally, I added a **pendulum swing** effect to the cable car, which makes the animation feel more natural and alive. +This iterative process taught me that small design changes — like color, movement, and alignment — can significantly affect how intuitive and enjoyable an interface feels. What started as a plain functional clock gradually became a **playful and interactive visualization of time**, something that feels less like “telling time” and more like “watching time travel.” diff --git a/Lab 2/cable car clock.png b/Lab 2/cable car clock.png new file mode 100644 index 0000000000..a3f0fa24df Binary files /dev/null and b/Lab 2/cable car clock.png differ diff --git a/Lab 2/clock.jpg b/Lab 2/clock.jpg new file mode 100644 index 0000000000..a2539a478e Binary files /dev/null and b/Lab 2/clock.jpg differ diff --git a/Lab 2/color.mp4 b/Lab 2/color.mp4 new file mode 100644 index 0000000000..dc2089805b Binary files /dev/null and b/Lab 2/color.mp4 differ diff --git a/Lab 2/image.jpg b/Lab 2/image.jpg new file mode 100644 index 0000000000..29b3e88bbf Binary files /dev/null and b/Lab 2/image.jpg differ diff --git a/Lab 2/image.mp4 b/Lab 2/image.mp4 new file mode 100644 index 0000000000..0cac8084b5 Binary files /dev/null and b/Lab 2/image.mp4 differ diff --git a/Lab 2/interactive clock.jpg b/Lab 2/interactive clock.jpg new file mode 100644 index 0000000000..c126af7268 Binary files /dev/null and b/Lab 2/interactive clock.jpg differ diff --git a/Lab 2/screen_cable_clock.py b/Lab 2/screen_cable_clock.py new file mode 100644 index 0000000000..a371c6e75d --- /dev/null +++ b/Lab 2/screen_cable_clock.py @@ -0,0 +1,190 @@ +import time +import datetime +import digitalio +import board +import math +from PIL import Image, ImageDraw, ImageFont, ImageFilter +import adafruit_rgb_display.st7789 as st7789 + +# ================= Display Setup ================= +cs_pin = digitalio.DigitalInOut(board.D5) +dc_pin = digitalio.DigitalInOut(board.D25) +reset_pin = None +BAUDRATE = 64000000 +spi = board.SPI() + +disp = st7789.ST7789( + spi, + cs=cs_pin, + dc=dc_pin, + rst=reset_pin, + baudrate=BAUDRATE, + width=135, + height=240, + x_offset=53, + y_offset=40, +) + +height = disp.width +width = disp.height +rotation = 90 + +image = Image.new("RGB", (width, height)) +draw = ImageDraw.Draw(image) + +font_title = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 18) +font_car = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 20) # slightly bigger font +font_small = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 12) + +backlight = digitalio.DigitalInOut(board.D22) +backlight.switch_to_output() +backlight.value = True + +# ================= Helper Functions ================= +def gradient_colors(trips, t, max_trips=48): + """Compute top and bottom colors for gradient background, with dynamic wave""" + intensity = min(trips / max_trips, 1.0) + wave = (math.sin(t / 5) + 1) / 2 # oscillates between 0 and 1 every ~10s + + r_top = int(200 + 40 * intensity + 30 * wave) + g_top = int(40 * (1 - intensity) + 30 * wave) + b_top = int(100 + 40 * wave) + + r_bottom = int(100 + 120 * intensity + 40 * wave) + g_bottom = int(20 * (1 - intensity) + 20 * wave) + b_bottom = int(40 + 60 * wave) + return (r_top, g_top, b_top), (r_bottom, g_bottom, b_bottom) + + +def draw_vertical_gradient(draw, width, height, top_color, bottom_color): + """Draw vertical gradient background""" + for y in range(height): + ratio = y / height + r = int(top_color[0] * (1 - ratio) + bottom_color[0] * ratio) + g = int(top_color[1] * (1 - ratio) + bottom_color[1] * ratio) + b = int(top_color[2] * (1 - ratio) + bottom_color[2] * ratio) + draw.line((0, y, width, y), fill=(r, g, b)) + + +def draw_shadow(base_img, car_x, car_y, car_w, car_h, roof_height, blur_radius=8, offset=6): + """Draw soft shadow under cable car""" + shadow = Image.new("RGBA", base_img.size, (0, 0, 0, 0)) + shadow_draw = ImageDraw.Draw(shadow) + + shadow_draw.polygon( + [(car_x+offset, car_y+offset), + (car_x+car_w+offset, car_y+offset), + (car_x+car_w-10+offset, car_y-roof_height+offset), + (car_x+10+offset, car_y-roof_height+offset)], + fill=(0, 0, 0, 120) + ) + + shadow_draw.rounded_rectangle( + (car_x+offset, car_y+offset, car_x+car_w+offset, car_y+car_h+offset), + radius=8, + fill=(0, 0, 0, 120) + ) + + shadow = shadow.filter(ImageFilter.GaussianBlur(blur_radius)) + base_img.paste(shadow, (0, 0), shadow) + + +def rotate_point(x, y, cx, cy, angle): + """Rotate point (x,y) around (cx,cy) by angle (radians)""" + s, c = math.sin(angle), math.cos(angle) + x -= cx + y -= cy + x_new = x * c - y * s + y_new = x * s + y * c + return x_new + cx, y_new + cy + +# ================= Main Loop ================= +while True: + draw.rectangle((0, 0, width, height), outline=0, fill=(0, 0, 0)) + + now = datetime.datetime.now() + t = time.time() + minutes_since_midnight = now.hour * 60 + now.minute + now.second / 60 + trips_float = minutes_since_midnight / 30 + + # Dynamic gradient background + top_color, bottom_color = gradient_colors(trips_float, t) + draw_vertical_gradient(draw, width, height, top_color, bottom_color) + + # ================= Draw Cable Car ================= + cable_y = height // 4 + 10 + draw.line((0, cable_y, width, cable_y), fill=(0, 100, 255), width=4) + + rope_length = 25 + car_w, car_h = 60, 40 + car_y = cable_y + rope_length + car_x_offset = 60 + + # Car moves back and forth every 30 minutes + seconds_in_day = now.hour * 3600 + now.minute * 60 + now.second + seconds_in_cycle = seconds_in_day % 1800 + if seconds_in_cycle < 900: + progress = seconds_in_cycle / 900 + car_x = int(progress * (width - car_x_offset)) + else: + progress = (seconds_in_cycle - 900) / 900 + car_x = int((1 - progress) * (width - car_x_offset)) + + # Sway angle for pendulum effect + sway_angle = math.sin(t * 2) * math.radians(5) # ±5 degrees sway + pivot_x = car_x + car_w // 2 + pivot_y = cable_y + + car_color = (135, 206, 250) + + # Rope with sway + rope_end_x, rope_end_y = rotate_point(pivot_x, car_y, pivot_x, pivot_y, sway_angle) + draw.line((pivot_x, pivot_y, rope_end_x, rope_end_y), fill=(0, 100, 255), width=3) + + # Shadow + draw_shadow(image, int(rope_end_x - car_w/2), int(rope_end_y), car_w, car_h, roof_height=10) + + # Roof + roof_height = 10 + roof_coords = [ + (rope_end_x - car_w/2, rope_end_y), + (rope_end_x + car_w/2, rope_end_y), + (rope_end_x + car_w/2 - 10, rope_end_y - roof_height), + (rope_end_x - car_w/2 + 10, rope_end_y - roof_height), + ] + rotated_roof = [rotate_point(x, y, pivot_x, pivot_y, sway_angle) for x, y in roof_coords] + draw.polygon(rotated_roof, fill=car_color) + + # Car body + car_body_coords = (rope_end_x - car_w/2, rope_end_y, rope_end_x + car_w/2, rope_end_y + car_h) + x0, y0 = rotate_point(car_body_coords[0], car_body_coords[1], pivot_x, pivot_y, sway_angle) + x1, y1 = rotate_point(car_body_coords[2], car_body_coords[3], pivot_x, pivot_y, sway_angle) + draw.rounded_rectangle((x0, y0, x1, y1), radius=8, fill=car_color) + + # Windows + win_margin = 6 + win_w, win_h = 12, 14 + for i in range(2): + wx = rope_end_x - car_w/2 + win_margin + i*(win_w + 14) + wy = rope_end_y + 10 + wx0, wy0 = rotate_point(wx, wy, pivot_x, pivot_y, sway_angle) + wx1, wy1 = rotate_point(wx+win_w, wy+win_h, pivot_x, pivot_y, sway_angle) + draw.rounded_rectangle((wx0, wy0, wx1, wy1), radius=4, fill=(200,230,250), outline=(255,255,255)) + draw.line((wx0+2, wy0+2, wx1-2, wy0+6), fill=(255,255,255), width=2) + + # Trip count inside car (centered text, rotated with car) + trips_text = f"{trips_float:.2f}" + text_bbox = draw.textbbox((0, 0), trips_text, font=font_car) + text_w = text_bbox[2] - text_bbox[0] + text_h = text_bbox[3] - text_bbox[1] + + text_x, text_y = rotate_point(rope_end_x, rope_end_y + car_h//2, pivot_x, pivot_y, sway_angle) + draw.text((text_x - text_w//2, text_y - text_h//2 - 2), trips_text, font=font_car, fill=(0,0,0)) + + # ================= Labels ================= + draw.text((5, 5), "Cable round trips", font=font_title, fill=(255, 255, 255)) + draw.text((5, height-18), "1 round trip = 30 min", font=font_small, fill=(255, 255, 255)) + + # Update display + disp.image(image, rotation) + time.sleep(0.05) diff --git a/Lab 2/screen_clock.py b/Lab 2/screen_clock.py index aa3bfb93ec..a43b43cb2c 100644 --- a/Lab 2/screen_clock.py +++ b/Lab 2/screen_clock.py @@ -4,6 +4,7 @@ import board from PIL import Image, ImageDraw, ImageFont import adafruit_rgb_display.st7789 as st7789 +from time import strftime # Configuration for CS and DC pins (these are FeatherWing defaults on M0/M4): cs_pin = digitalio.DigitalInOut(board.D5) @@ -60,12 +61,16 @@ backlight.switch_to_output() backlight.value = True + while True: # Draw a black filled box to clear the image. - draw.rectangle((0, 0, width, height), outline=0, fill=400) + draw.rectangle((0, 0, width, height), outline=0, fill=(0, 0, 0)) #TODO: Lab 2 part D work should be filled in here. You should be able to look in cli_clock.py and stats.py + current_time = strftime("%m/%d/%Y %H:%M:%S") + draw.text((10, height//2 - 10), current_time, font=font, fill=(255, 255, 255)) # Display image. disp.image(image, rotation) time.sleep(1) + diff --git a/Lab 2/text.jpg b/Lab 2/text.jpg new file mode 100644 index 0000000000..42712023db Binary files /dev/null and b/Lab 2/text.jpg differ diff --git a/Lab 3/Dialogue performing.m4a b/Lab 3/Dialogue performing.m4a new file mode 100644 index 0000000000..541abdf07e Binary files /dev/null and b/Lab 3/Dialogue performing.m4a differ diff --git a/Lab 3/README.md b/Lab 3/README.md index 25c6970386..ce7beee876 100644 --- a/Lab 3/README.md +++ b/Lab 3/README.md @@ -1,314 +1,295 @@ # Chatterboxes -**NAMES OF COLLABORATORS HERE** -[![Watch the video](https://user-images.githubusercontent.com/1128669/135009222-111fe522-e6ba-46ad-b6dc-d1633d21129c.png)](https://www.youtube.com/embed/Q8FWzLMobx0?start=19) +**Joy Sun (Jiayi Sun) and Sandy Zhan (Huiying Zhan)** -In this lab, we want you to design interaction with a speech-enabled device--something that listens and talks to you. This device can do anything *but* control lights (since we already did that in Lab 1). First, we want you first to storyboard what you imagine the conversational interaction to be like. Then, you will use wizarding techniques to elicit examples of what people might say, ask, or respond. We then want you to use the examples collected from at least two other people to inform the redesign of the device. +## Part 1. +### Text to Speech -We will focus on **audio** as the main modality for interaction to start; these general techniques can be extended to **video**, **haptics** or other interactive mechanisms in the second part of the Lab. +In this part of lab, we are going to start peeking into the world of audio on your Pi! -## Prep for Part 1: Get the Latest Content and Pick up Additional Parts +\*\***Write your own shell file to use your favorite of these TTS engines to have your Pi greet you by name.**\*\* +(This shell file should be saved to your own repo for this lab.) -Please check instructions in [prep.md](prep.md) and complete the setup before class on Wednesday, Sept 23rd. +See my implementation here: [greet_joy.sh](./speech-scripts/greet_joy.sh) -### Pick up Web Camera If You Don't Have One +--- +### Speech to Text -Students who have not already received a web camera will receive their [Logitech C270 Webcam](https://www.amazon.com/Logitech-Desktop-Widescreen-Calling-Recording/dp/B004FHO5Y6/ref=sr_1_3?crid=W5QN79TK8JM7&dib=eyJ2IjoiMSJ9.FB-davgIQ_ciWNvY6RK4yckjgOCrvOWOGAG4IFaH0fczv-OIDHpR7rVTU8xj1iIbn_Aiowl9xMdeQxceQ6AT0Z8Rr5ZP1RocU6X8QSbkeJ4Zs5TYqa4a3C_cnfhZ7_ViooQU20IWibZqkBroF2Hja2xZXoTqZFI8e5YnF_2C0Bn7vtBGpapOYIGCeQoXqnV81r2HypQNUzFQbGPh7VqjqDbzmUoloFA2-QPLa5lOctA.L5ztl0wO7LqzxrIqDku9f96L9QrzYCMftU_YeTEJpGA&dib_tag=se&keywords=webcam%2Bc270&qid=1758416854&sprefix=webcam%2Bc270%2Caps%2C125&sr=8-3&th=1) and bluetooth speaker on Wednesday at the beginning of lab. If you cannot make it to class this week, please contact the TAs to ensure you get these. +\*\***Write your own shell file that verbally asks for a numerical based input (such as a phone number, zipcode, number of pets, etc) and records the answer the respondent provides.**\*\* -### Get the Latest Content +See my number speech detection script here: [ask_number.sh](./speech-scripts/ask_number.sh) -As always, pull updates from the class Interactive-Lab-Hub to both your Pi and your own GitHub repo. There are 2 ways you can do so: +### Picture 1: Number Speech Record +Interactive Speech AI -**\[recommended\]**Option 1: On the Pi, `cd` to your `Interactive-Lab-Hub`, pull the updates from upstream (class lab-hub) and push the updates back to your own GitHub repo. You will need the *personal access token* for this. -``` -pi@ixe00:~$ cd Interactive-Lab-Hub -pi@ixe00:~/Interactive-Lab-Hub $ git pull upstream Fall2025 -pi@ixe00:~/Interactive-Lab-Hub $ git add . -pi@ixe00:~/Interactive-Lab-Hub $ git commit -m "get lab3 updates" -pi@ixe00:~/Interactive-Lab-Hub $ git push -``` +### 🤖 NEW: AI-Powered Conversations with Ollama -Option 2: On your your own GitHub repo, [create pull request](https://github.com/FAR-Lab/Developing-and-Designing-Interactive-Devices/blob/2022Fall/readings/Submitting%20Labs.md) to get updates from the class Interactive-Lab-Hub. After you have latest updates online, go on your Pi, `cd` to your `Interactive-Lab-Hub` and use `git pull` to get updates from your own GitHub repo. +\*\***Try creating a simple voice interaction that combines speech recognition, Ollama processing, and text-to-speech output. Document what you built and how users responded to it.**\*\* -## Part 1. -### Setup +### Picture 2: Interactive Speech +Interactive Speech AI -Activate your virtual environment -``` -pi@ixe00:~$ cd Interactive-Lab-Hub -pi@ixe00:~/Interactive-Lab-Hub $ cd Lab\ 3 -pi@ixe00:~/Interactive-Lab-Hub/Lab 3 $ python3 -m venv .venv -pi@ixe00:~/Interactive-Lab-Hub $ source .venv/bin/activate -(.venv)pi@ixe00:~/Interactive-Lab-Hub $ -``` +## My System Components -Run the setup script -```(.venv)pi@ixe00:~/Interactive-Lab-Hub $ pip install -r requirements.txt ``` +### 1. Speech Recognition +- We use the Python [`speech_recognition`](https://pypi.org/project/SpeechRecognition/) library to capture audio from a microphone. +- The system supports selecting a preferred microphone (e.g., Logitech C270 HD Webcam) automatically, with fallback to the default device if the preferred one is unavailable. +- Listening mechanism configuration: + - Listen for a maximum of **15 seconds** per input. + - Automatically stop listening if the user is silent for **2 seconds**. +- This allows quick capture of short user queries while avoiding long idle recordings. -Next, run the setup script to install additional text-to-speech dependencies: -``` -(.venv)pi@ixe00:~/Interactive-Lab-Hub/Lab 3 $ ./setup.sh -``` -### Text to Speech +### 2. Ollama AI Processing +- Captured speech is converted to text via **Google Speech Recognition**. +- The text is sent as a prompt to the Ollama model (`phi3:mini`) via its REST API. +- Query timeout is **3 minutes** to allow for complex responses, with progress messages shown to the user. +- Ollama generates a textual response based on the user’s input. -In this part of lab, we are going to start peeking into the world of audio on your Pi! -We will be using the microphone and speaker on your webcamera. In the directory is a folder called `speech-scripts` containing several shell scripts. `cd` to the folder and list out all the files by `ls`: +### 3. Text-to-Speech (TTS) Output +- Response text is converted into speech using **espeak**. +- Special character handling: + - Characters like `–` or `—` are replaced or removed to prevent TTS errors. +- Speech playback is fully completed before the next listening session begins, preventing the microphone from capturing the assistant’s own voice. -``` -pi@ixe00:~/speech-scripts $ ls -Download festival_demo.sh GoogleTTS_demo.sh pico2text_demo.sh -espeak_demo.sh flite_demo.sh lookdave.wav -``` -You can run these shell files `.sh` by typing `./filename`, for example, typing `./espeak_demo.sh` and see what happens. Take some time to look at each script and see how it works. You can see a script by typing `cat filename`. For instance: +### 4. Concurrency and Flow Control +- Blocking calls are used for espeak, ensuring each TTS playback completes before listening starts again. +- Prevents overlapping input/output, ensuring only the user’s voice is recognized. -``` -pi@ixe00:~/speech-scripts $ cat festival_demo.sh -#from: https://elinux.org/RPi_Text_to_Speech_(Speech_Synthesis)#Festival_Text_to_Speech -``` -You can test the commands by running -``` -echo "Just what do you think you're doing, Dave?" | festival --tts -``` -Now, you might wonder what exactly is a `.sh` file? -Typically, a `.sh` file is a shell script which you can execute in a terminal. The example files we offer here are for you to figure out the ways to play with audio on your Pi! +## User Experience and Feedback +- Users can speak naturally in English and receive almost immediate responses. +- Responses are read aloud in a clear voice, creating a conversational feel. +- Long or complex queries are handled gracefully, with a 3-minute timeout for AI responses. +- Special character handling ensures TTS errors do not interrupt the interaction. +- Users found the system intuitive, with quick response times and smooth audio feedback. -You can also play audio files directly with `aplay filename`. Try typing `aplay lookdave.wav`. -\*\***Write your own shell file to use your favorite of these TTS engines to have your Pi greet you by name.**\*\* -(This shell file should be saved to your own repo for this lab.) +## Conclusion +This project demonstrates a fully functional voice interaction loop combining: ---- -Bonus: -[Piper](https://github.com/rhasspy/piper) is another fast neural based text to speech package for raspberry pi which can be installed easily through python with: -``` -pip install piper-tts -``` -and used from the command line. Running the command below the first time will download the model, concurrent runs will be faster. -``` -echo 'Welcome to the world of speech synthesis!' | piper \ - --model en_US-lessac-medium \ - --output_file welcome.wav -``` -Check the file that was created by running `aplay welcome.wav`. Many more languages are supported and audio can be streamed dirctly to an audio output, rather than into an file by: - -``` -echo 'This sentence is spoken first. This sentence is synthesized while the first sentence is spoken.' | \ - piper --model en_US-lessac-medium --output-raw | \ - aplay -r 22050 -f S16_LE -t raw - -``` - -### Speech to Text +- **Speech recognition** +- **AI processing via Ollama** +- **Text-to-speech output** -Next setup speech to text. We are using a speech recognition engine, [Vosk](https://alphacephei.com/vosk/), which is made by researchers at Carnegie Mellon University. Vosk is amazing because it is an offline speech recognition engine; that is, all the processing for the speech recognition is happening onboard the Raspberry Pi. +The system effectively handles microphone selection, ambient noise calibration, Unicode-safe TTS, and sequential listening/speaking, resulting in a responsive and user-friendly voice assistant prototype. -Make sure you're running in your virtual environment with the dependencies already installed: -``` -source .venv/bin/activate -``` -Test if vosk works by transcribing text: +### Storyboard +\*\***Post your storyboard and diagram here.**\*\* -``` -vosk-transcriber -i recorded_mono.wav -o test.txt -``` +### Verplank diagram +We designed a **Smart Mirror Outfit Assistant** that recommends daily outfits based on **weather, temperature, and special occasions**. The interaction is through **voice input** and **speech + visual overlay** output. -You can use vosk with the microphone by running -``` -python test_microphone.py -m en -``` +1. **Morning Start** +Storyboard Situation 1 ---- -Bonus: -[Whisper](https://openai.com/index/whisper/) is a neural network–based speech-to-text (STT) model developed and open-sourced by OpenAI. Compared to Vosk, Whisper generally achieves higher accuracy, particularly on noisy audio and diverse accents. It is available in multiple model sizes; for edge devices such as the Raspberry Pi 5 used in this class, the tiny.en model runs with reasonable latency even without a GPU. +2. **Special Occasion** +Storyboard Situation 2 -By contrast, Vosk is more lightweight and optimized for running efficiently on low-power devices like the Raspberry Pi. The choice between Whisper and Vosk depends on your scenario: if you need higher accuracy and can afford slightly more compute, Whisper is preferable; if your priority is minimal resource usage, Vosk may be a better fit. +3. **Misunderstanding Branch** +Storyboard Situation 3 -In this class, we provide two Whisper options: A quantized 8-bit faster-whisper model for speed, and the standard Whisper model. Try them out and compare the trade-offs. -Make sure you're in the Lab 3 directory with your virtual environment activated: -``` -cd ~/Interactive-Lab-Hub/Lab\ 3/speech-scripts -source ../.venv/bin/activate -``` +\*\***Please describe and document your process.**\*\* +## 🌳 Dialogue Tree: Smart Mirror Outfit Assistant +### Panel 1 – Morning Start (Weather-based Recommendation) -Then test the Whisper models: -``` -python whisper_try.py -``` -and +**User**: "Mirror, what should I wear today?" +→ *System checks today’s weather* -``` -python faster_whisper_try.py -``` -\*\***Write your own shell file that verbally asks for a numerical based input (such as a phone number, zipcode, number of pets, etc) and records the answer the respondent provides.**\*\* +#### If Rainy 🌧️ +- **System**: "It’s rainy today. I recommend wearing a raincoat and waterproof shoes." +- **User**: "Should I also bring an umbrella?" +- **System**: "Yes, carrying an umbrella will keep you dry." -### 🤖 NEW: AI-Powered Conversations with Ollama +#### If Snowy ❄️ +- **System**: "It’s snowing today. I suggest a down jacket, warm boots, and gloves." +- **User**: "Can I wear sneakers instead?" +- **System**: "Sneakers may get wet in the snow. Boots would be safer." -Want to add intelligent conversation capabilities to your voice projects? **Ollama** lets you run AI models locally on your Raspberry Pi for sophisticated dialogue without requiring internet connectivity! +#### If Sunny & Hot ☀️ +- **System**: "It’s sunny and warm, around 28 degrees. I recommend a T-shirt and shorts." +- **User**: "Do I need sunglasses?" +- **System**: "Yes, sunglasses will help protect your eyes from the bright sun." -#### Quick Start with Ollama +#### If Cold but Clear 🌤️ +- **System**: "It’s 12 degrees but clear. I suggest wearing long pants and a jacket." -**Installation** (takes ~5 minutes): -```bash -# Install Ollama -curl -fsSL https://ollama.com/install.sh | sh +--- -# Download recommended model for Pi 5 -ollama pull phi3:mini +### Panel 2 – Special Occasion (Context-based Recommendation) -# Install system dependencies for audio (required for pyaudio) -sudo apt-get update -sudo apt-get install -y portaudio19-dev python3-dev +**User**: "I have a job interview today." +→ *System applies formal attire rules* -# Create separate virtual environment for Ollama (due to pyaudio conflicts) -cd ollama/ -python3 -m venv ollama_venv -source ollama_venv/bin/activate +- **System**: "For a formal occasion, I recommend a dark suit with a white shirt. That will look more professional." +- **User**: "Should I wear a tie?" + - If **Yes** → "A simple dark tie will make your outfit more polished." + - If **No** → "That’s fine, but with a tie you’ll look more formal." -# Install Python dependencies in separate environment -pip install -r ollama_requirements.txt -``` -#### Ready-to-Use Scripts +- **User**: "Can you show me how it looks?" +- **System**: *[Overlay suit + tie on reflection]* -We've created three Ollama integration scripts for different use cases: +#### Alternative Occasions +- **If Dinner Date** → "A smart-casual outfit would be good, like a blazer with chinos." +- **If Sports Event** → "Casual wear is fine — maybe a hoodie and sneakers." -**1. Basic Demo** - Learn how Ollama works: -```bash -python3 ollama_demo.py -``` +--- -**2. Voice Assistant** - Full speech-to-text + AI + text-to-speech: -```bash -python3 ollama_voice_assistant.py -``` +### Panel 3 – Misunderstanding Branch (Error Handling) -**3. Web Interface** - Beautiful web-based chat with voice options: -```bash -python3 ollama_web_app.py -# Then open: http://localhost:5000 -``` +**User**: "Recommend a pair of shoes." +→ *ASR Misheard* → "Recommend a skirt." -#### Integration in Your Projects +- **System (clarification strategy)**: "Did you mean skirt or shoes?" +- **User**: "Shoes." +- **System**: "Great. Do you want formal shoes or casual shoes?" + - If **Formal** → "Black leather shoes would match your suit perfectly." + - If **Casual** → "White sneakers would go well with a casual outfit." + -Simple example to add AI to any project: -```python -import requests +### Acting out the dialogue -def ask_ai(question): - response = requests.post( - "http://localhost:11434/api/generate", - json={"model": "phi3:mini", "prompt": question, "stream": False} - ) - return response.json().get('response', 'No response') +### 🎧 Dialogue Audio Recording +You can listen to the acted-out dialogue here: +[Dialogue performing.m4a](./Dialogue%20performing.m4a) -# Use it anywhere! -answer = ask_ai("How should I greet users?") -``` +\*\***Describe if the dialogue seemed different than what you imagined when it was acted out, and how.**\*\* +### Reflection: Differences Between Imagined and Real Dialogue -**📖 Complete Setup Guide**: See `OLLAMA_SETUP.md` for detailed instructions, troubleshooting, and advanced usage! +When we acted out the dialogue, it turned out to be quite different from the imagined dialogue tree. -\*\***Try creating a simple voice interaction that combines speech recognition, Ollama processing, and text-to-speech output. Document what you built and how users responded to it.**\*\* +#### Structured vs. Natural Flow +- **Imagined version**: Highly structured, with predefined conditions (rainy, snowy, sunny, cold). + - User asked short, direct questions like *“Should I also bring an umbrella?”* or *“Do I need sunglasses?”*. + - The flow assumed clear, logical branches. -### Serving Pages +- **Real version**: User spoke more naturally and unpredictably. + - Example: *“Today I would like to go out. What kind of suit would you recommend?”* + - This shifted the topic toward **activity-based clothing** rather than just weather. + - System (played by partner) adapted by asking about the **occasion**, leading to discussions about **sportswear, tennis skirts, and even color preferences**. -In Lab 1, we served a webpage with flask. In this lab, you may find it useful to serve a webpage for the controller on a remote device. Here is a simple example of a webserver. +#### Personalization and Context +- Real dialogue introduced **unexpected context and personalization**. + - Example: detecting closet inventory (*“blue and pink skirts”*) and giving **tailored recommendations**. +- This personalization was **not considered** in the original dialogue flow. -``` -pi@ixe00:~/Interactive-Lab-Hub/Lab 3 $ python server.py - * Serving Flask app "server" (lazy loading) - * Environment: production - WARNING: This is a development server. Do not use it in a production deployment. - Use a production WSGI server instead. - * Debug mode: on - * Running on http://0.0.0.0:5000/ (Press CTRL+C to quit) - * Restarting with stat - * Debugger is active! - * Debugger PIN: 162-573-883 -``` -From a remote browser on the same network, check to make sure your webserver is working by going to `http://:5000`. You should be able to see "Hello World" on the webpage. +#### Key Insights +- **Imagined script**: Useful as a starting point to structure logic. +- **Real interaction**: Showed that actual users bring in: + - Personal preferences + - Casual, varied language + - Follow-up questions beyond the rigid tree -### Storyboard +**Conclusion**: The acting-out exercise highlighted the importance of **flexibility, personalization, and error-handling** in real system design, beyond what a fixed dialogue tree can capture. -Storyboard and/or use a Verplank diagram to design a speech-enabled device. (Stuck? Make a device that talks for dogs. If that is too stupid, find an application that is better than that.) -\*\***Post your storyboard and diagram here.**\*\* +# Lab 3 Part 2 -Write out what you imagine the dialogue to be. Use cards, post-its, or whatever method helps you develop alternatives or group responses. +For Part 2, you will redesign the interaction with the speech-enabled device using the data collected, as well as feedback from part 1. -\*\***Please describe and document your process.**\*\* +## Prototype your system +### 1.New Story Board +#### Situation 1: Choose the Type +Choose the Type -### Acting out the dialogue +#### Situation 2: Choose the Outfit +Choose the Outfit -Find a partner, and *without sharing the script with your partner* try out the dialogue you've designed, where you (as the device designer) act as the device you are designing. Please record this interaction (for example, using Zoom's record feature). +### 2.Document how the system works +### System Documentation 1: Wizard of oz +#### 1.Overview -\*\***Describe if the dialogue seemed different than what you imagined when it was acted out, and how.**\*\* +This project is an interactive demo that runs on a Raspberry Pi and connects a SparkFun Qwiic Joystick to a web interface. The system streams joystick data to a browser in real time and allows the user to convert typed text into spoken audio on the Pi. The implementation uses a lightweight Flask web server with Socket.IO for bi-directional messaging, and espeak for local text-to-speech output. -### Wizarding with the Pi (optional) -In the [demo directory](./demo), you will find an example Wizard of Oz project. In that project, you can see how audio and sensor data is streamed from the Pi to a wizard controller that runs in the browser. You may use this demo code as a template. By running the `app.py` script, you can see how audio and sensor data (Adafruit MPU-6050 6-DoF Accel and Gyro Sensor) is streamed from the Pi to a wizard controller that runs in the browser `http://:5000`. You can control what the system says from the controller as well! +#### 2.What we achieved -\*\***Describe if the dialogue seemed different than what you imagined, or when acted out, when it was wizarded, and how.**\*\* +- Provide a clean, low-latency demo of hardware → web interaction. -# Lab 3 Part 2 +- Replace an accelerometer-based input with a Qwiic Joystick as the primary interaction device. -For Part 2, you will redesign the interaction with the speech-enabled device using the data collected, as well as feedback from part 1. +- Keep audio output stable and deterministic (avoid flaky real-time microphone streaming). -## Prep for Part 2 +#### 3.How it works +This prototype demonstrates an interactive dialogue system between a user and a smart mirror, implemented on a Raspberry Pi. -1. What are concrete things that could use improvement in the design of your device? For example: wording, timing, anticipation of misunderstandings... -2. What are other modes of interaction _beyond speech_ that you might also use to clarify how to interact? -3. Make a new storyboard, diagram and/or script based on these reflections. +The interaction is simulated through a hardware joystick and a web interface: -## Prototype your system +- The joystick acts as the user’s physical input device, representing gestures or presence in front of the mirror. +When the joystick is moved or pressed, its real-time position data are captured by the Raspberry Pi and transmitted to the browser via a lightweight Flask + Socket.IO server. +This simulates how the mirror might sense or respond to a user’s movements or actions. -The system should: -* use the Raspberry Pi -* use one or more sensors -* require participants to speak to it. +- The web interface represents the mirror’s intelligence and speech output. +The browser allows text input, which mimics what the mirror would “say” in response to the user. +When a line of text is entered, it is sent to the Raspberry Pi through a WebSocket connection. +The Pi immediately converts the text into speech using the built-in text-to-speech engine (espeak), producing an audible reply through the speaker — just as a smart mirror might talk back to the user. -*Document how the system works* +**Through this setup, the system forms a closed interaction loop:** -*Include videos or screencaptures of both the system and the controller.* +1.User gesture or presence → sensed by the joystick and visualized on the web page. -
- Submission Cleanup Reminder (Click to Expand) - - **Before submitting your README.md:** - - This readme.md file has a lot of extra text for guidance. - - Remove all instructional text and example prompts from this file. - - You may either delete these sections or use the toggle/hide feature in VS Code to collapse them for a cleaner look. - - Your final submission should be neat, focused on your own work, and easy to read for grading. - - This helps ensure your README.md is clear professional and uniquely yours! -
+2.Mirror response → simulated by typed text, spoken aloud by the Raspberry Pi. -## Test the system -Try to get at least two people to interact with your system. (Ideally, you would inform them that there is a wizard _after_ the interaction, but we recognize that can be hard.) +This prototype effectively recreates a two-way conversational experience between a user and an intelligent mirror, without relying on heavy machine-learning or cloud components. It provides a simple, tangible framework to explore timing, responsiveness, and interaction design in human-mirror communication. -Answer the following: +### System Documentation 2: Outfit Recommendation Assistant -### What worked well about the system and what didn't? -\*\**your answer here*\*\* +#### 1. Overview +This project is an interactive outfit recommendation system implemented on a Raspberry Pi using a SparkFun Qwiic Joystick as the primary input device. The system runs locally and provides real-time voice feedback to guide users through outfit themes and style options. Each joystick movement or press triggers context-aware spoken responses generated through Piper text-to-speech. The prototype simulates a “smart mirror” that can not only recommend outfits but also engage in short, spoken dialogues with the user through a secondary voice interaction script. The goal is to explore embodied interaction and conversational timing in tangible, voice-driven interfaces. -### What worked well about the controller and what didn't? +#### 2. What We Achieved +- Created a fully functional **joystick-based voice interface** that allows users to browse outfit themes (Sports, Social, Color, Interview, Daily) and listen to style options without a screen. +- Implemented **robust motion recognition** and threshold calibration for smooth, low-latency navigation on the Raspberry Pi. +- Integrated **text-to-speech (Piper → aplay fallback to Espeak)** for consistent and clear local audio playback. +- Added **voice Q&A mode** triggered by joystick press or mid-hold gesture, enabling interactive question–answer conversations powered by `voice_interaction_router.py`. -\*\**your answer here*\*\* +#### 3. How It Works +The system establishes a continuous interaction loop between the user’s physical gestures and the Raspberry Pi’s audio responses: -### What lessons can you take away from the WoZ interactions for designing a more autonomous version of the system? +- **Hardware Input (Joystick):** + The SparkFun Qwiic Joystick provides horizontal, vertical, and button data over I²C. At startup, the script samples joystick values to determine the neutral center and automatically sets sensitivity thresholds. Moving the joystick **up or down** changes the outfit theme, while **left or right** cycles through the available options within that theme. Each action immediately triggers a voice message in the format: + `recommend type: . Option . ` -\*\**your answer here*\*\* +- **Speech Output (TTS):** + When a new theme or option is selected, the system converts the corresponding text from `outfits.py` into audio using Piper TTS, then plays it through ALSA (`aplay`). If Piper fails, Espeak automatically handles fallback playback to maintain responsiveness. +- **Voice Interaction:** + Pressing the joystick (or holding it steady at center for 1.2s, if button readout fails) launches `voice_interaction_router.py`. This script processes user speech or text queries, determines relevant outfit logic from `outfits.py`, and returns a spoken reply. The exchange mimics natural dialogue—like asking a mirror, “What should I wear for sports?”—and receiving a contextual spoken answer. -### How could you use your system to create a dataset of interaction? What other sensing modalities would make sense to capture? +**Overall Flow:** +1. User performs a joystick gesture → Raspberry Pi detects motion and determines action type. +2. Corresponding text prompt is selected from `outfits.py`. +3. Text is synthesized into speech via Piper and played aloud. +4. Optional voice Q&A extends the dialogue, forming a closed feedback loop between physical input and auditory output. -\*\**your answer here*\*\* +This architecture enables a fully local, screenless, and conversational prototype—illustrating how a tangible interface can blend physical gestures with responsive, expressive voice output to create a personalized interactive mirror experience. +### 3.Include videos or screencaptures of both the system and the controller. +[Watch the Mirror Interaction video here](https://youtu.be/myvbhKIoJT8) +[Watch the User Conversation video here](https://youtu.be/t43ZWVyX4Zk) + + + + +## Test the system + +### What worked well about the system and what didn't? +The system successfully guided both participants through theme and outfit selection with clear audio feedback. The voice output was natural and the “recommend type” prefix helped them understand the context of each option. The participants found the up/down and left/right navigation intuitive once they heard the first few lines. However, there was occasional delay in audio playback, especially when Piper was generating longer sentences. One participant also noticed that quick joystick movements were sometimes ignored, suggesting that the input thresholds could be more responsive. + +### What worked well about the controller and what didn't? +The Qwiic Joystick offered smooth analog control and provided a clear sense of directionality for switching between options. The physical form made it easy for users to remember which axis changed themes versus options. The press-to-speak action worked well for triggering the voice Q&A mode, but some users pressed too briefly, leading to missed triggers due to hardware debounce. The fallback “hold in the middle” interaction helped prevent deadlocks, but users didn’t always realize it was an intentional feature. + +### What lessons can you take away from the WoZ interactions for designing a more autonomous version of the system? +From the Wizard-of-Oz sessions, it became clear that the system benefits from short, conversational feedback to keep users engaged. Participants tended to wait for confirmation after each action, implying that a fully autonomous version should generate quick acknowledgments such as “Got it” or “Switching to Social theme.” It also showed that adaptive timing—pausing slightly after playback before accepting new input—would make interactions feel smoother. A more autonomous version could combine user intent prediction (based on repeated joystick patterns) with contextual language responses for a more human-like experience. + +### How could you use your system to create a dataset of interaction? What other sensing modalities would make sense to capture? +The system could log joystick positions, button states, timestamps, and recognized speech transcripts to create a dataset of multimodal interactions. Each session could store pairs of physical input sequences and spoken responses, enabling supervised learning of user intent patterns. Adding a small microphone array could capture tone and hesitation, while a camera could record facial expressions or gaze direction to analyze engagement. These additional sensing modalities would allow future models to learn richer mappings between gesture, voice, and emotional state, supporting more adaptive and autonomous dialogue behavior. diff --git a/Lab 3/choose the outfit.jpg b/Lab 3/choose the outfit.jpg new file mode 100644 index 0000000000..d195795553 Binary files /dev/null and b/Lab 3/choose the outfit.jpg differ diff --git a/Lab 3/choose the type.jpg b/Lab 3/choose the type.jpg new file mode 100644 index 0000000000..41b0bdca7d Binary files /dev/null and b/Lab 3/choose the type.jpg differ diff --git a/Lab 3/demo/app.py b/Lab 3/demo/app.py index cd4f75affe..2e6c2bd6c4 100644 --- a/Lab 3/demo/app.py +++ b/Lab 3/demo/app.py @@ -1,49 +1,52 @@ import eventlet eventlet.monkey_patch() -from flask import Flask, Response,render_template -from flask_socketio import SocketIO, send, emit -from subprocess import Popen, call - -import time -import board -import busio -#import adafruit_mpu6050 -from adafruit_msa3xx import MSA311 -import json -import socket +from flask import Flask, render_template +from flask_socketio import SocketIO, emit +from subprocess import call +import socket import signal import sys -from queue import Queue - -i2c = busio.I2C(board.SCL, board.SDA) -#mpu = adafruit_mpu6050.MPU6050(i2c) -msa = MSA311(i2c) +import qwiic_joystick + +joystick = qwiic_joystick.QwiicJoystick() +if not joystick.connected: + print("Joystick not connected! Please check wiring (SDA/SCL).") +else: + joystick.begin() + print("Joystick connected successfully.") hostname = socket.gethostname() -hardware = 'plughw:2,0' +# Flask + SocketIO app = Flask(__name__) socketio = SocketIO(app) -audio_stream = Popen("/usr/bin/cvlc alsa://"+hardware+" --sout='#transcode{vcodec=none,acodec=mp3,ab=256,channels=2,samplerate=44100,scodec=none}:http{mux=mp3,dst=:8080/}' --no-sout-all --sout-keep", shell=True) @socketio.on('speak') -def handel_speak(val): +def handle_speak(val): + print(f"Speaking: {val}") call(f"espeak '{val}'", shell=True) @socketio.on('connect') def test_connect(): - print('connected') - emit('after connect', {'data':'Lets dance'}) + print('Client connected') + emit('after connect', {'data': 'Joystick ready!'}) @socketio.on('ping-gps') def handle_message(val): - # print(mpu.acceleration) - emit('pong-gps', msa.acceleration) + joystick.horizontal + joystick.vertical + joystick.button + data = { + 'horizontal': joystick.horizontal, + 'vertical': joystick.vertical, + 'button': joystick.button + } + emit('pong-gps', data) @app.route('/') def index(): @@ -51,13 +54,10 @@ def index(): def signal_handler(sig, frame): print('Closing Gracefully') - audio_stream.terminate() sys.exit(0) signal.signal(signal.SIGINT, signal_handler) - if __name__ == "__main__": + print("Starting Flask + Joystick server at port 5000...") socketio.run(app, host='0.0.0.0', port=5000) - - diff --git a/Lab 3/demo/requirements.txt b/Lab 3/demo/requirements.txt index 0efd65fb6c..e1508e85ae 100644 --- a/Lab 3/demo/requirements.txt +++ b/Lab 3/demo/requirements.txt @@ -2,19 +2,18 @@ Adafruit-Blinka==6.4.0 adafruit-circuitpython-busdevice==5.0.6 adafruit-circuitpython-msa301==1.3.0 adafruit-circuitpython-mpu6050==1.1.6 -adafruit-circuitpython-msa301==1.3.0 adafruit-circuitpython-register==1.9.5 Adafruit-PlatformDetect==3.3.0 Adafruit-PureIO==1.1.8 bidict==0.21.2 click==7.1.2 dnspython==1.16.0 -eventlet==0.31.0 +eventlet==0.33.3 Flask==1.1.2 Flask-SocketIO==5.0.1 -gevent==21.1.2 +gevent==22.10.2 gevent-websocket==0.10.1 -greenlet==1.0.0 +greenlet==2.0.2 itsdangerous==1.1.0 Jinja2==2.11.3 MarkupSafe==1.1.1 diff --git a/Lab 3/demo/templates/index.html b/Lab 3/demo/templates/index.html index 6b9af227a7..c205534c49 100644 --- a/Lab 3/demo/templates/index.html +++ b/Lab 3/demo/templates/index.html @@ -1,30 +1,67 @@ - - - - - - - - - - Document + + + + + + + + + + Joystick Dashboard -
-
- - -
-
- - +

🎮 Joystick Dashboard

+
+

Horizontal: --

+

Vertical: --

+

Button: --

+
+ +
+ + + +

+ + + diff --git a/Lab 3/interactive speech ai.jpg b/Lab 3/interactive speech ai.jpg new file mode 100644 index 0000000000..02985f45bd Binary files /dev/null and b/Lab 3/interactive speech ai.jpg differ diff --git a/Lab 3/ollama/ollama_voice_assistant.py b/Lab 3/ollama/ollama_voice_assistant.py index 3e5dfc6afd..217e6ed7df 100644 --- a/Lab 3/ollama/ollama_voice_assistant.py +++ b/Lab 3/ollama/ollama_voice_assistant.py @@ -1,213 +1,118 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Ollama Voice Assistant for Lab 3 -Interactive voice assistant using speech recognition, Ollama AI, and text-to-speech - -Dependencies: -- ollama (API client) -- speech_recognition -- pyaudio -- pyttsx3 or espeak +- Speech input via microphone +- Query Ollama (up to 3 minutes) +- Speak response using espeak (Unicode safe, BLOCKING) """ import speech_recognition as sr -import subprocess import requests -import json +import subprocess import time import sys -import threading -from queue import Queue - -# Set UTF-8 encoding for output -if sys.stdout.encoding != 'UTF-8': - import codecs - sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict') -if sys.stderr.encoding != 'UTF-8': - import codecs - sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict') - -try: - import pyttsx3 - TTS_ENGINE = 'pyttsx3' -except ImportError: - TTS_ENGINE = 'espeak' - print("pyttsx3 not available, using espeak for TTS") - -class OllamaVoiceAssistant: - def __init__(self, model_name="phi3:mini", ollama_url="http://localhost:11434"): - self.model_name = model_name - self.ollama_url = ollama_url - self.recognizer = sr.Recognizer() - self.microphone = sr.Microphone() - - # Initialize TTS - if TTS_ENGINE == 'pyttsx3': - self.tts_engine = pyttsx3.init() - self.tts_engine.setProperty('rate', 150) # Speed of speech - - # Test Ollama connection - self.test_ollama_connection() - - # Adjust for ambient noise - print("Adjusting for ambient noise... Please wait.") - with self.microphone as source: - self.recognizer.adjust_for_ambient_noise(source) - print("Ready for conversation!") - - def test_ollama_connection(self): - """Test if Ollama is running and the model is available""" - try: - response = requests.get(f"{self.ollama_url}/api/tags") - if response.status_code == 200: - models = response.json().get('models', []) - model_names = [m['name'] for m in models] - if self.model_name in model_names: - print(f"Ollama is running with {self.model_name} model") - else: - print(f"Model {self.model_name} not found. Available models: {model_names}") - if model_names: - self.model_name = model_names[0] - print(f"Using {self.model_name} instead") - else: - raise Exception("Ollama API not responding") - except Exception as e: - print(f"Error connecting to Ollama: {e}") - print("Make sure Ollama is running: 'ollama serve'") - sys.exit(1) - - def speak(self, text): - """Convert text to speech""" - # Clean text to avoid encoding issues - clean_text = text.encode('ascii', 'ignore').decode('ascii') - print(f"Assistant: {clean_text}") - - if TTS_ENGINE == 'pyttsx3': - self.tts_engine.say(clean_text) - self.tts_engine.runAndWait() - else: - # Use espeak as fallback - subprocess.run(['espeak', clean_text], check=False) - - def listen(self): - """Listen for speech and convert to text""" - try: - print("Listening...") - with self.microphone as source: - # Listen for audio with timeout - audio = self.recognizer.listen(source, timeout=5, phrase_time_limit=10) - - print("Recognizing...") - # Use Google Speech Recognition (free) - text = self.recognizer.recognize_google(audio) - print(f"You said: {text}") - return text.lower() - - except sr.WaitTimeoutError: - print("No speech detected, timing out...") - return None - except sr.UnknownValueError: - print("Could not understand audio") - return None - except sr.RequestError as e: - print(f"Error with speech recognition service: {e}") - return None - - def query_ollama(self, prompt, system_prompt=None): - """Send a query to Ollama and get response""" - try: - data = { - "model": self.model_name, - "prompt": prompt, - "stream": False - } - - if system_prompt: - data["system"] = system_prompt - - response = requests.post( - f"{self.ollama_url}/api/generate", - json=data, - timeout=30 - ) - - if response.status_code == 200: - result = response.json() - return result.get('response', 'Sorry, I could not generate a response.') - else: - return f"Error: Ollama API returned status {response.status_code}" - - except requests.exceptions.Timeout: - return "Sorry, the response took too long. Please try again." - except Exception as e: - return f"Error communicating with Ollama: {e}" - - def run_conversation(self): - """Main conversation loop""" - print("\nOllama Voice Assistant Started!") - print("Say 'hello' to start, 'exit' or 'quit' to stop") - print("=" * 50) - - # System prompt to make the assistant more conversational - system_prompt = """You are a helpful voice assistant. Keep your responses concise and conversational, - typically 1-2 sentences. Be friendly and engaging. You are running on a Raspberry Pi as part of an - interactive device design lab.""" - - self.speak("Hello! I'm your Ollama voice assistant. How can I help you today?") - - while True: - try: - # Listen for user input - user_input = self.listen() - - if user_input is None: - continue - - # Check for exit commands - if any(word in user_input for word in ['exit', 'quit', 'bye', 'goodbye']): - self.speak("Goodbye! Have a great day!") - break - - # Check for greeting - if any(word in user_input for word in ['hello', 'hi', 'hey']): - self.speak("Hello! What would you like to talk about?") - continue - - # Send to Ollama for processing - print("Thinking...") - response = self.query_ollama(user_input, system_prompt) - - # Speak the response - self.speak(response) - - except KeyboardInterrupt: - print("\nConversation interrupted by user") - self.speak("Goodbye!") - break - except Exception as e: - print(f"Unexpected error: {e}") - self.speak("Sorry, I encountered an error. Let's try again.") -def main(): - """Main function to run the voice assistant""" - print("Starting Ollama Voice Assistant...") - - # Check if required dependencies are available +# Ensure stdout uses utf-8 to avoid encoding errors +sys.stdout.reconfigure(encoding='utf-8') + +# Ollama configuration +OLLAMA_URL = "http://localhost:11434" +DEFAULT_MODEL = "phi3:mini" + +def get_microphone_index(): + """Find and return the index of the preferred microphone.""" + mic_list = sr.Microphone.list_microphone_names() + print("Detected microphones:") + for idx, name in enumerate(mic_list): + print(f"{idx}: {name}") + + # Prefer Logitech C270 HD Webcam + for i, name in enumerate(mic_list): + if "C270 HD WEBCAM" in name: + print(f"Using preferred device: {name} (index={i})") + return i + + # Fallback to default + print("C270 microphone not found, using default device") + return 0 + +def query_ollama(prompt, model=DEFAULT_MODEL, timeout=180): + """Send prompt to Ollama and return response (up to 3 minutes).""" try: - import speech_recognition - import requests - except ImportError as e: - print(f"Missing dependency: {e}") - print("Please install with: pip install speechrecognition requests pyaudio") - return - - # Create and run the assistant + response = requests.post( + f"{OLLAMA_URL}/api/generate", + json={"model": model, "prompt": prompt, "stream": False}, + timeout=timeout + ) + + if response.status_code == 200: + return response.json().get("response", "No response generated") + else: + return f"Error: Ollama returned status {response.status_code}" + + except requests.exceptions.Timeout: + return "Sorry, the response took too long. Please try again." + except Exception as e: + return f"Error: {str(e)}" + +def speak_text(text): + """Convert text to speech using espeak (BLOCKING, safe Unicode).""" try: - assistant = OllamaVoiceAssistant() - assistant.run_conversation() + # Replace problematic characters + safe_text = text.replace('–', '-').replace('—', '-').replace('…', '...') + + safe_text = ''.join(c if ord(c) < 128 else ' ' for c in safe_text) + + subprocess.run(['espeak', '-v', 'en', safe_text], check=False, encoding='utf-8', timeout=120) + except subprocess.TimeoutExpired: + print("TTS timeout exceeded") except Exception as e: - print(f"Failed to start assistant: {e}") + print(f"TTS Error: {e}") + + +def main(): + """Main loop: listen → recognize → query → speak.""" + device_index = get_microphone_index() + r = sr.Recognizer() + + try: + with sr.Microphone(device_index=device_index) as source: + print("Calibrating microphone for ambient noise...") + r.adjust_for_ambient_noise(source, duration=1) + print("Calibration complete") + + print("Voice assistant ready. Speak into the microphone.") + print("Press Ctrl+C to stop.") + + while True: + try: + print("\nListening...") + # Listen for up to 15 seconds, end after 2s silence + audio = r.listen(source, timeout=15, phrase_time_limit=15) + + print("Recognizing speech...") + user_text = r.recognize_google(audio, language="en-US") + print(f"You said: {user_text}") + + print("Querying Ollama (this may take up to 3 minutes)...") + ai_response = query_ollama(user_text) + print(f"Ollama: {ai_response}") + + # BLOCKING speak: next listen starts only after TTS finishes + speak_text(ai_response) + + except sr.WaitTimeoutError: + print("No speech detected in time.") + except sr.UnknownValueError: + print("Could not understand audio.") + except sr.RequestError as e: + print(f"Speech Recognition service error: {e}") + except Exception as e: + print(f"Error: {e}") + time.sleep(1) + + except KeyboardInterrupt: + print("\nExiting voice assistant...") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/Lab 3/ollama/ollama_web_app.py b/Lab 3/ollama/ollama_web_app.py index a5b896f6c5..4b30d72233 100644 --- a/Lab 3/ollama/ollama_web_app.py +++ b/Lab 3/ollama/ollama_web_app.py @@ -1,98 +1,130 @@ #!/usr/bin/env python3 """ Ollama Flask Web Interface for Lab 3 -Web-based voice assistant using Ollama +Web-based voice assistant using Ollama AI -This extends the existing Flask app in demo/app.py to include Ollama integration +This script extends a standard Flask app with: +- Ollama AI integration +- REST API chat endpoint +- WebSocket support for real-time chat +- Text-to-speech via espeak """ import eventlet -eventlet.monkey_patch() +eventlet.monkey_patch() # Patch standard libraries to work with eventlet from flask import Flask, Response, render_template, request, jsonify from flask_socketio import SocketIO, send, emit import requests -import json import subprocess -import os +# Initialize Flask app app = Flask(__name__) +# Enable WebSocket support with CORS allowed from all origins socketio = SocketIO(app, cors_allowed_origins="*") +# -------------------- # Ollama configuration -OLLAMA_URL = "http://localhost:11434" -DEFAULT_MODEL = "phi3:mini" +# -------------------- +OLLAMA_URL = "http://localhost:11434" # Ollama local server +DEFAULT_MODEL = "phi3:mini" # Default AI model +# -------------------- +# Helper functions +# -------------------- def query_ollama(prompt, model=DEFAULT_MODEL): - """Query Ollama and return response""" + """ + Query Ollama API and return the AI's response. + Handles timeout and general exceptions. + """ try: response = requests.post( f"{OLLAMA_URL}/api/generate", - json={ - "model": model, - "prompt": prompt, - "stream": False - }, + json={"model": model, "prompt": prompt, "stream": False}, timeout=30 ) - if response.status_code == 200: return response.json().get('response', 'No response generated') else: return f"Error: Ollama returned status {response.status_code}" - except requests.exceptions.Timeout: return "Sorry, the response took too long. Please try again." except Exception as e: return f"Error: {str(e)}" def speak_text(text): - """Text-to-speech using espeak""" + """ + Convert text to speech using espeak. + Runs as a subprocess to speak asynchronously. + """ try: subprocess.run(['espeak', f'"{text}"'], shell=True, check=False) except Exception as e: print(f"TTS Error: {e}") +# -------------------- +# Flask routes +# -------------------- @app.route('/') def index(): - """Main web interface""" + """Render the main web interface""" return render_template('ollama_chat.html') @app.route('/api/chat', methods=['POST']) def chat_api(): - """REST API endpoint for chat""" + """ + REST API endpoint for chat messages. + Expects JSON with a 'message' field. + Returns AI response in JSON format. + """ data = request.get_json() user_message = data.get('message', '') - if not user_message: return jsonify({'error': 'No message provided'}), 400 - - # Query Ollama + response = query_ollama(user_message) - - return jsonify({ - 'user_message': user_message, - 'ai_response': response - }) + return jsonify({'user_message': user_message, 'ai_response': response}) + +@app.route('/status') +def status(): + """ + Endpoint to check Ollama server status. + Returns connected models and current model. + """ + try: + response = requests.get(f"{OLLAMA_URL}/api/tags", timeout=5) + if response.status_code == 200: + models = response.json().get('models', []) + return jsonify({ + 'status': 'connected', + 'models': [m['name'] for m in models], + 'current_model': DEFAULT_MODEL + }) + else: + return jsonify({'status': 'error', 'message': 'Ollama not responding'}), 500 + except Exception as e: + return jsonify({'status': 'error', 'message': str(e)}), 500 +# -------------------- +# WebSocket handlers +# -------------------- @socketio.on('chat_message') def handle_chat_message(data): - """Handle chat message via WebSocket""" + """ + Handle real-time chat messages from client via WebSocket. + Sends AI response back using 'ai_response' event. + """ user_message = data.get('message', '') - if user_message: - # Query Ollama ai_response = query_ollama(user_message) - - # Send response back to client - emit('ai_response', { - 'user_message': user_message, - 'ai_response': ai_response - }) + emit('ai_response', {'user_message': user_message, 'ai_response': ai_response}) @socketio.on('speak_request') def handle_speak_request(data): - """Handle text-to-speech request""" + """ + Handle TTS requests from client. + Converts provided text to speech using espeak. + """ text = data.get('text', '') if text: speak_text(text) @@ -100,40 +132,24 @@ def handle_speak_request(data): @socketio.on('voice_chat') def handle_voice_chat(data): - """Handle voice chat request (text in, voice out)""" + """ + Handle combined voice chat requests: + - User sends text + - Ollama generates a response + - Response is spoken aloud via TTS + - Response sent back to client + """ user_message = data.get('message', '') - if user_message: - # Query Ollama ai_response = query_ollama(user_message) - - # Speak the response speak_text(ai_response) - - # Send response to client - emit('voice_response', { - 'user_message': user_message, - 'ai_response': ai_response - }) - -@app.route('/status') -def status(): - """Check Ollama status""" - try: - response = requests.get(f"{OLLAMA_URL}/api/tags", timeout=5) - if response.status_code == 200: - models = response.json().get('models', []) - return jsonify({ - 'status': 'connected', - 'models': [m['name'] for m in models], - 'current_model': DEFAULT_MODEL - }) - else: - return jsonify({'status': 'error', 'message': 'Ollama not responding'}), 500 - except Exception as e: - return jsonify({'status': 'error', 'message': str(e)}), 500 + emit('voice_response', {'user_message': user_message, 'ai_response': ai_response}) +# -------------------- +# Main entry point +# -------------------- if __name__ == '__main__': - print("🚀 Starting Ollama Flask Web Interface...") + # Print UTF-8 safe startup message + print("Starting Ollama Flask Web Interface...") print("Open your browser to http://localhost:5000") - socketio.run(app, host='0.0.0.0', port=5000, debug=True) \ No newline at end of file + socketio.run(app, host='0.0.0.0', port=5000, debug=True) diff --git a/Lab 3/speech number recording.jpg b/Lab 3/speech number recording.jpg new file mode 100644 index 0000000000..072d07df9e Binary files /dev/null and b/Lab 3/speech number recording.jpg differ diff --git a/Lab 3/speech-scripts/ask_number.sh b/Lab 3/speech-scripts/ask_number.sh new file mode 100644 index 0000000000..3caad9e701 --- /dev/null +++ b/Lab 3/speech-scripts/ask_number.sh @@ -0,0 +1,4 @@ +#!/bin/bash +source ../.venv/bin/activate + +python3 record_number.py diff --git a/Lab 3/speech-scripts/greet_joy.sh b/Lab 3/speech-scripts/greet_joy.sh new file mode 100644 index 0000000000..e88cee5434 --- /dev/null +++ b/Lab 3/speech-scripts/greet_joy.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +NAME="Joy" +echo "Hello, $NAME! Welcome to Lab Three." | espeak -s 170 -v en-us diff --git a/Lab 3/speech-scripts/record_number.py b/Lab 3/speech-scripts/record_number.py new file mode 100644 index 0000000000..e853c16cc4 --- /dev/null +++ b/Lab 3/speech-scripts/record_number.py @@ -0,0 +1,36 @@ +import whisper +import sounddevice as sd +import numpy as np +import tempfile +import wavio +import os +import re + +os.system('espeak "Please say a number now"') + +duration = 5 +samplerate = 16000 + +print("Recording... Speak now!") +recording = sd.rec(int(duration * samplerate), samplerate=samplerate, channels=1, dtype='int16') +sd.wait() +print("Recording finished!") + +with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + wavio.write(f.name, recording, samplerate, sampwidth=2) + wav_path = f.name + +model = whisper.load_model("tiny.en") +result = model.transcribe(wav_path) +text = result.get("text", "") + +numbers_only = "".join(re.findall(r'\d+', text)) + +if numbers_only: + print("You said (numbers only):", numbers_only) + os.system(f'espeak "You said {numbers_only}"') +else: + print("No numbers detected.") + os.system('espeak "No numbers detected"') + +os.remove(wav_path) diff --git a/Lab 3/storyboard1.jpg b/Lab 3/storyboard1.jpg new file mode 100644 index 0000000000..9befc4ece8 Binary files /dev/null and b/Lab 3/storyboard1.jpg differ diff --git a/Lab 3/storyboard2.jpg b/Lab 3/storyboard2.jpg new file mode 100644 index 0000000000..1031f6bc73 Binary files /dev/null and b/Lab 3/storyboard2.jpg differ diff --git a/Lab 3/storyboard3.jpg b/Lab 3/storyboard3.jpg new file mode 100644 index 0000000000..739d03836d Binary files /dev/null and b/Lab 3/storyboard3.jpg differ diff --git a/Lab 4/Design1.jpeg b/Lab 4/Design1.jpeg new file mode 100644 index 0000000000..bfcd56ccbb Binary files /dev/null and b/Lab 4/Design1.jpeg differ diff --git a/Lab 4/Design2.jpeg b/Lab 4/Design2.jpeg new file mode 100644 index 0000000000..f5b86bb88b Binary files /dev/null and b/Lab 4/Design2.jpeg differ diff --git a/Lab 4/Design3.jpeg b/Lab 4/Design3.jpeg new file mode 100644 index 0000000000..3953daf0e1 Binary files /dev/null and b/Lab 4/Design3.jpeg differ diff --git a/Lab 4/Design4.jpeg b/Lab 4/Design4.jpeg new file mode 100644 index 0000000000..184c049b5d Binary files /dev/null and b/Lab 4/Design4.jpeg differ diff --git a/Lab 4/Design5.jpeg b/Lab 4/Design5.jpeg new file mode 100644 index 0000000000..cb1bf62bab Binary files /dev/null and b/Lab 4/Design5.jpeg differ diff --git a/Lab 4/Handcraft1.jpeg b/Lab 4/Handcraft1.jpeg new file mode 100644 index 0000000000..19ce63def1 Binary files /dev/null and b/Lab 4/Handcraft1.jpeg differ diff --git a/Lab 4/Handcraft2.jpeg b/Lab 4/Handcraft2.jpeg new file mode 100644 index 0000000000..b4da00c231 Binary files /dev/null and b/Lab 4/Handcraft2.jpeg differ diff --git a/Lab 4/Prototype1.jpg b/Lab 4/Prototype1.jpg new file mode 100644 index 0000000000..4bcaa0a152 Binary files /dev/null and b/Lab 4/Prototype1.jpg differ diff --git a/Lab 4/Prototype2.jpg b/Lab 4/Prototype2.jpg new file mode 100644 index 0000000000..6f0dc350fb Binary files /dev/null and b/Lab 4/Prototype2.jpg differ diff --git a/Lab 4/Prototype3.jpg b/Lab 4/Prototype3.jpg new file mode 100644 index 0000000000..51b7f4a899 Binary files /dev/null and b/Lab 4/Prototype3.jpg differ diff --git a/Lab 4/Prototype4.jpg b/Lab 4/Prototype4.jpg new file mode 100644 index 0000000000..b900243d45 Binary files /dev/null and b/Lab 4/Prototype4.jpg differ diff --git a/Lab 4/README.md b/Lab 4/README.md index afbb46ed98..799cb23e1c 100644 --- a/Lab 4/README.md +++ b/Lab 4/README.md @@ -1,4 +1,5 @@ + # Ph-UI!!!
@@ -11,11 +12,12 @@ - Your final submission should be neat, focused on your own work, and easy to read for grading. This helps ensure your README.md is clear, professional, and uniquely yours! -
---- -## Lab 4 Deliverables +--- + +
+

Lab 4 Deliverables

 ### Part 1 (Week 1) **Submit the following for Part 1:** @@ -53,14 +55,15 @@ - Reflection on what you learned and next steps --- +
## Lab Overview -**NAMES OF COLLABORATORS HERE** - +Jiayi Sun and Huiying Zhan For lab this week, we focus both on sensing, to bring in new modes of input into your devices, as well as prototyping the physical look and feel of the device. You will think about the physical form the device needs to perform the sensing as well as present the display or feedback about what was sensed. -## Part 1 Lab Preparation +
+

Part 1 Lab Preparation

 ### Get the latest content: As always, pull updates from the class Interactive-Lab-Hub to both your Pi and your own GitHub repo. As we discussed in the class, there are 2 ways you can do so: @@ -99,8 +102,10 @@ Option 3: (preferred) use the Github.com interface to update the changes. (We do offer shared cutting board, cutting tools, and markers on the class cart during the lab, so do not worry if you don't have them!) +
-## Deliverables \& Submission for Lab 4 +
+

Deliverables \& Submission for Lab 4

 The deliverables for this lab are, writings, sketches, photos, and videos that show what your prototype: * "Looks like": shows how the device should look, feel, sit, weigh, etc. @@ -111,7 +116,7 @@ For submission, the readme.md page for this lab should be edited to include the * Upload any materials that explain what you did, into your lab 4 repository, and link them in your lab 4 readme.md. * Link your Lab 4 readme.md in your main Interactive-Lab-Hub readme.md. * Labs are due on Mondays, make sure to submit your Lab 4 readme.md to Canvas. - +
## Lab Overview @@ -130,7 +135,8 @@ F) [Record the interaction](#part-f) ## The Report (Part 1: A-D, Part 2: E-F) -### Quick Start: Python Environment Setup +
+

Quick Start: Python Environment Setup

 1. **Create and activate a virtual environment in Lab 4:** ```bash @@ -147,10 +153,11 @@ F) [Record the interaction](#part-f) python blinkatest.py ``` If you see "Hello blinka!", your setup is correct. If not, follow the troubleshooting steps in the file or ask for help. +
### Part A -### Capacitive Sensing, a.k.a. Human-Twizzler Interaction - +
+

Capacitive Sensing, a.k.a. Human-Twizzler Interaction

 We want to introduce you to the [capacitive sensor](https://learn.adafruit.com/adafruit-mpr121-gator) in your kit. It's one of the most flexible input devices we are able to provide. At boot, it measures the capacitance on each of the 12 contacts. Whenever that capacitance changes, it considers it a user touch. You can attach any conductive material. In your kit, you have copper tape that will work well, but don't limit yourself! In the example below, we use Twizzlers--you should pick your own objects. @@ -168,11 +175,15 @@ These Twizzlers are connected to pads 6 and 10. When you run the code and touch Twizzler 10 touched! Twizzler 6 touched! ``` +
+ +[Watch the Capacitive Sensing Test video here](https://youtu.be/Zkkp4hpUmT0) + ### Part B ### More sensors - -#### Light/Proximity/Gesture sensor (APDS-9960) +
+

Light/Proximity/Gesture sensor (APDS-9960)

 We here want you to get to know this awesome sensor [Adafruit APDS-9960](https://www.adafruit.com/product/3595). It is capable of sensing proximity, light (also RGB), and gesture! @@ -191,9 +202,15 @@ Connect it to your pi with Qwiic connector and try running the three example scr ``` You can go the the [Adafruit GitHub Page](https://github.com/adafruit/Adafruit_CircuitPython_APDS9960) to see more examples for this sensor! +
+ +[Watch the Proximity Test video here](https://youtu.be/O3EXj9QbpGg) +[Watch the Gesture Test video here](https://youtu.be/lSCrs3jGfG8) +[Watch the Color Test video here](https://youtu.be/w0VfGBpko4k) -#### Rotary Encoder +
+

Rotary Encoder

 A rotary encoder is an electro-mechanical device that converts the angular position to analog or digital output signals. The [Adafruit rotary encoder](https://www.adafruit.com/product/4991#technical-details) we ordered for you came with separate breakout board and encoder itself, that is, they will need to be soldered if you have not yet done so! We will be bringing the soldering station to the lab class for you to use, also, you can go to the MakerLAB to do the soldering off-class. Here is some [guidance on soldering](https://learn.adafruit.com/adafruit-guide-excellent-soldering/preparation) from Adafruit. When you first solder, get someone who has done it before (ideally in the MakerLAB environment). It is a good idea to review this material beforehand so you know what to look at.

@@ -210,10 +227,12 @@ Connect it to your pi with Qwiic connector and try running the example script, i ``` You can go to the [Adafruit Learn Page](https://learn.adafruit.com/adafruit-i2c-qt-rotary-encoder/python-circuitpython) to learn more about the sensor! The sensor actually comes with an LED (neo pixel): Can you try lighting it up? +

-#### Joystick - +[Watch the Encoder Test video here](https://youtu.be/-VMnZd_K9Kw) +
+

Joystick

 A [joystick](https://www.sparkfun.com/products/15168) can be used to sense and report the input of the stick for it pivoting angle or direction. It also comes with a button input!

@@ -227,10 +246,12 @@ Connect it to your pi with Qwiic connector and try running the example script to ``` You can go to the [SparkFun GitHub Page](https://github.com/sparkfun/Qwiic_Joystick_Py) to learn more about the sensor! +

-#### Distance Sensor - +[Watch the Joystick Test video here](https://youtu.be/3UiWbI6_ZkE) +
+

Distance Sensor

 Earlier we have asked you to play with the proximity sensor, which is able to sense objects within a short distance. Here, we offer [Sparkfun Proximity Sensor Breakout](https://www.sparkfun.com/products/15177), With the ability to detect objects up to 20cm away.

@@ -245,23 +266,90 @@ Connect it to your pi with Qwiic connector and try running the example script to ``` You can go to the [SparkFun GitHub Page](https://github.com/sparkfun/Qwiic_Proximity_Py) to learn more about the sensor and see other examples +

+ +[Watch the Distance Sensor Test video here](https://youtu.be/49AnOakKdbU) ### Part C -### Physical considerations for sensing +
+

Physical considerations for sensing

 +Usually, sensors need to be positioned in specific locations or orientations to make them useful for their application. Now that you've tried a bunch of the sensors, pick one that you would like to use, and an application where you use the output of that sensor for an interaction. For example, you can use a distance sensor to measure someone's height if you position it overhead and get them to stand under it. +
+ +### Smart Hygiene Dispenser +Detects hand proximity to automatically dispense sanitizer and supports gesture control to switch modes. +

+ Smart Hygiene Dispenser +

+### Breathing Trainer +Synchronizes light brightness with the user's breathing rhythm and turns off once the user falls asleep. +

+ Breathing Trainer +

-Usually, sensors need to be positioned in specific locations or orientations to make them useful for their application. Now that you've tried a bunch of the sensors, pick one that you would like to use, and an application where you use the output of that sensor for an interaction. For example, you can use a distance sensor to measure someone's height if you position it overhead and get them to stand under it. +### Touchless Music Controller +Uses simple hand gestures to switch, pause, and resume songs without physical contact. +

+ Touchless Music Controller +

+### Distance Mirror +Displays real-time weather and time when the user approaches, and hides them when the user moves away. +

+ Distance Mirror +

-**\*\*\*Draw 5 sketches of different ways you might use your sensor, and how the larger device needs to be shaped in order to make the sensor useful.\*\*\*** +### Gesture Lamp +Allows users to adjust light intensity and switch lighting modes through up–down and left–right hand waves. +

+ Gesture Lamp +

+ +--- +### What questions do these sketches raise? + +While sketching these five concepts, we realized that many of them share similar challenges around gesture detection, sensing range, and user feedback. +Some key questions that emerged are: + +#### 1. Sensor sensitivity and detection range +How far should the user's hand be for the APDS-9960 to reliably detect gestures? +Will small or fast movements be missed or misclassified? -**\*\*\*What are some things these sketches raise as questions? What do you need to physically prototype to understand how to anwer those questions?\*\*\*** +#### 2. Sensor placement and orientation +Does the sensor need to face directly toward the user, or can it work at an angle or behind a surface like acrylic? +How much does the physical mounting affect recognition accuracy? -**\*\*\*Pick one of these designs to prototype.\*\*\*** +#### 3. Environmental interference +Will ambient light, sunlight, or reflections interfere with the proximity or gesture readings? + +#### 4. User feedback and learnability +How can the system communicate successful detection? +Do users need audio, visual, or haptic cues to understand when the system has responded? + +#### 5. Safety and usability in real contexts +In touchless designs (like the dispenser or music controller), how do we avoid false triggers or unwanted activations when people simply pass by? + + +### What do we need to prototype to answer those questions? +To explore these issues, we will need to physically prototype and test the following aspects: +1. Vary the distance and angle between the user's hand and the sensor to measure detection accuracy. +2. Test gesture recognition (up, down, left, right) in different lighting environments. +3. Integrate visual or sound feedback (like LEDs or audio tones) to evaluate whether users can easily understand system responses. +4. Experiment with different enclosure materials (cardboard, plastic, translucent cover) to see how they affect proximity sensing. +5. Observe how users interact with the device naturally, identifying when false detections occur. + + +### Prototype Choice +We decided to prototype the gesture-controlled music player using the APDS-9960 sensor. +This design feels both fun and practical—it allows users to control music playback without touching any buttons, which is useful when their hands are busy or unclean (for example, while cooking or studying). + +With this prototype, we want to explore how well the sensor can recognize left and right gestures for switching songs, and near/far gestures for pausing and resuming playback. It will also help us understand how users perceive the responsiveness of the system and whether adding visual or audio feedback (like an LED flash or short tone) improves the overall interaction experience. ### Part D -### Physical considerations for displaying information and housing parts +
+

Physical considerations for displaying information and housing parts

 @@ -300,20 +388,111 @@ Here is an example: Think about how you want to present the information about what your sensor is sensing! Design a paper display for your project that communicates the state of the Pi and a sensor. Ideally you should design it so that you can slide the Pi out to work on the circuit or programming, and then slide it back in and reattach a few wires to be back in operation. - -**\*\*\*Sketch 5 designs for how you would physically position your display and any buttons or knobs needed to interact with it.\*\*\*** +
+ +### Sketch 5 designs for how you would physically position your display and any buttons or knobs needed to interact with it. +#### Sketch 1 – Turntable +

+ Design1 +

+ +- The APDS-9960 sensor is positioned at the center of a turntable to detect left and right swipe gestures for switching songs. +- The larger device is designed as a flat record player, allowing users to interact naturally by moving their hands horizontally across the sensor. + +#### Sketch 2 – Song Player +

+ Design2 +

+ +- The focus of this design is how to display information through sound — using the speaker as the main medium to communicate playback status. +- Instead of visual feedback, music itself becomes the output, spreading through the speaker so that people nearby can hear and perceive the interaction. + +#### Sketch 3 – Interaction Diagram +

+ Design3 +

+ +- This diagram clarifies how hand gestures are recognized: + - Swipe right → Next song + - Swipe left → Previous song +- The Raspberry Pi connects to the APDS-9960 through I²C wiring, with enough open space around the sensor to avoid reflections or blocking. +- The first three sketches together describe a **single prototype concept** — an interactive vinyl record player that combines gesture-based control, sound output, and tangible form to create a better user experience. + +#### Sketch 4 – Wall-mounted Display Turntable +

+ Design4 +

+ +- The sensor are mounted on a vertical display turntable, fixed to the wall. +- Users can touch a simple “next/previous” button as backup control. +- Display on wall make device more public and interactive. + +#### Sketch 5 – Portable Pocket Player +

+ Design5 +

-**\*\*\*What are some things these sketches raise as questions? What do you need to physically prototype to understand how to anwer those questions?\*\*\*** +- A compact, pocket-sized version of the music player. +- Includes a screen, two small buttons for switching songs, and a rotary knob for volume control. +- The physical form focuses on mobility and personal use. -**\*\*\*Pick one of these display designs to integrate into your prototype.\*\*\*** +--- + +### What are some things these sketches raise as questions? What do you need to physically prototype to understand how to anwer those questions? +1. **Gesture Reliability:** + - How consistently does the APDS-9960 recognize left/right gestures under different lighting conditions and angles? + - Physical prototyping can help determine the optimal sensor placement and gesture distance. -**\*\*\*Explain the rationale for the design.\*\*\*** (e.g. Does it need to be a certain size or form or need to be able to be seen from a certain distance?) +2. **User Feedback and Visibility:** + - Can users easily understand which song is playing or what gesture was detected? + - Prototyping a screen layout or paper interface will help refine the feedback design. -Build a cardboard prototype of your design. +3. **Form and Accessibility:** + - How should the device’s size, height, and orientation change between tabletop, wall-mounted, and portable versions? + - Physical mockups will help test usability in different contexts. +4. **Interaction Comfort:** + - How natural does it feel to wave or swipe above the sensor? -**\*\*\*Document your rough prototype.\*\*\*** +5. **Aesthetic Integration:** + - How can the retro turntable appearance blend with the modern digital interaction? +--- +### Prototype +

+ prototype +

+ +[Watch the Prototype video here](https://youtu.be/M_O3RC8Aed4) + +The gesture-controlled music player is implemented in [`Lab 4/music_player.py`](https://github.com/hz764/Interactive-Lab-Hub-hz764/blob/Fall2025/Lab%204/music_player.py) + +### Explain the Rationale for the Design +- The design takes inspiration from a **vinyl record player**, emphasizing tangible interaction and visual familiarity. +- The circular base represents the turntable, while the **paper tonearm** mimics the motion of a real record player. +- The **APDS-9960 sensor** is positioned at the center, where users can naturally wave their hands to control playback. +- This placement ensures the gestures are **easily recognized** without obstruction and allows intuitive control from a short distance (5-10 cm). +- The overall **size and form** are chosen to be tabletop scale — large enough for comfortable hand motion, but compact enough to fit on a desk. +- The **speaker connection** allows music to be heard by everyone around, reinforcing the shared and social aspect of the design. + +### Document Your Rough Prototype +

+ Handcraft1 +

+

+ Handcraft2 +

+ +- The prototype is constructed using **cardboard and colored paper**, with the Raspberry Pi and APDS-9960 connected through I²C wiring. +- The tonearm is made from **lightweight cardboard**, and the base includes **aesthetic blue and black color contrast** to highlight the interaction zone. +- The Raspberry Pi handles audio playback and gesture recognition through a Python script. +- This rough prototype helps visualize how the **gesture input**, **sensor placement**, and **audio feedback** work together in a cohesive setup. +--- + +### User Feedback +- Early testers reported that the design was **visually clear and intuitive**, as the record-player metaphor made the interaction immediately understandable. +- However, they noted that **gesture sensitivity** could be improved — sometimes swipes were not recognized consistently, especially under bright lighting conditions. +- Users also suggested that adding **visual indicators or LED feedback** could make the interaction feel more responsive. # LAB PART 2 @@ -324,7 +503,9 @@ Following exploration and reflection from Part 1, complete the "looks like," "wo ### Part E - +
+Guidance (Click to Expand) + #### Chaining Devices and Exploring Interaction Effects For Part 2, you will design and build a fun interactive prototype using multiple inputs and outputs. This means chaining Qwiic and STEMMA QT devices (e.g., buttons, encoders, sensors, servos, displays) and/or combining with traditional breadboard prototyping (e.g., LEDs, buzzers, etc.). @@ -486,13 +667,137 @@ A servo motor is a rotary actuator that allows for precise control of angular po --- +
+ ### Part F ### Record +
+Guidance (Click to Expand) Document all the prototypes and iterations you have designed and worked on! Again, deliverables for this lab are writings, sketches, photos, and videos that show what your prototype: * "Looks like": shows how the device should look, feel, sit, weigh, etc. * "Works like": shows what the device can do * "Acts like": shows how a person would interact with the device +
+ + +## Looks Like + +### Picture 1: Prototype Overview +

+ Prototype1 +

+ +### Picture 2: Switch Songs +

+ Prototype2 +

+ +### Picture 3: Pause / Resume Songs +

+ Prototype3 +

+ +### Picture 4: Adjust Volume & Lights +

+ Prototype4 +

+ + +[▶️ Watch Volume Adjustment & Lights Video](https://youtu.be/YH-1EWWbGyo) +[▶️ Watch Pause & Switch Songs Video](https://youtu.be/auJRVmpbO4I) + +--- + +## Works Like + +### System Overview + +We built a **gesture-controlled music player** that combines multiple input and output devices through I²C chaining: + +**Inputs** +- **APDS-9960 Gesture Sensor** — detects left/right hand swipes to switch songs, and near/far gestures to pause or resume playback. +- **SparkFun Qwiic Button** — allows manual pause/play control as a backup input. +- **Adafruit Seesaw Rotary Encoder** — adjusts music volume with smooth physical rotation. + +**Outputs** +- **Qwiic GPIO Board + LEDs** — visual feedback showing current volume level. + - More LEDs light up as volume increases. + - Fewer LEDs indicate lower volume. +- **Speaker** — audio output from Raspberry Pi for real-time playback. + + +--- + +## Acts Like + +### Prototype Behavior + +- **Swipe Right:** Next song +- **Swipe Left:** Previous song +- **Near Gesture:** Pause playback +- **Far Gesture:** Resume playback +- **Press Qwiic Button:** Toggle play/pause manually +- **Rotate Encoder:** Adjust volume; LEDs dynamically show the loudness level + +This prototype merges **gesture, tactile, and visual** interaction modes to make the music player feel more expressive and intuitive. The gestures handle “remote” control, while the button and knob support **direct, physical control** for precision and reliability. + +--- + +## Physical Setup + +- The **APDS-9960 sensor** is centered on the top surface for consistent gesture detection within 5–10 cm. +- The **rotary encoder** and **Qwiic Button** are placed on the front edge for easy access. +- LEDs are arranged in a semicircle near the base, reinforcing the “turntable” look while providing visual feedback. +- All components are wired through the Qwiic I²C system to keep cabling neat and modular. + + +--- + +## User Test & Feedback + +### Test Setup +We conducted quick user testing with a few participants to observe: +1. Whether gestures were detected reliably in typical lighting. +2. How intuitive users found the button and encoder controls. +3. How well LED brightness/quantity communicated volume levels. + +### Feedback Highlights +- **Gesture Recognition:** + Most users could switch tracks easily with left/right swipes. However, near/far gestures for pause/play were sometimes missed under bright light or when the hand moved too fast. +- **Button Control:** + The button was clear and reassuring. Many users liked having a “backup” tactile option when gestures failed. +- **Rotary Encoder:** + Turning the knob to adjust volume felt natural and gave a sense of precision. + Users quickly understood that LED count = volume level. +- **Overall Experience:** + The combination of gesture and physical controls felt playful and interactive. + The LED feedback was engaging and made the system’s state visible at a glance. + +### Observations +- The APDS-9960 worked best within ~8 cm and perpendicular hand motion. +- Adding a short LED flash or sound cue after a recognized gesture could help confirm input success. +- A larger LED diffusion area would make the volume feedback look smoother. + +### Improvements for Next Iteration +- Optimize gesture detection thresholds in code for more stable recognition. +- Explore different LED color schemes (e.g., green to red gradient for volume). +- Miniaturize the prototype for better portability. + +--- + +## Reflection + +Through this multi-input/multi-output prototype, we learned how chaining Qwiic devices enables rich, layered interactions: + +- **Integration Challenge:** Coordinating multiple I²C devices required careful address management and timing. +- **User Experience Insight:** Combining gestures with physical input greatly improved usability—gestures are expressive, but buttons and knobs ensure reliability. +- **Fun Discovery:** The LED feedback added an emotional and visual dimension to sound, turning volume control into a small “light performance.” + +In short, the project showed how combining sensors, encoders, and LEDs can create **playful, multimodal experiences** that feel both modern and tangible. + +--- + diff --git a/Lab 4/Sensor1.jpeg b/Lab 4/Sensor1.jpeg new file mode 100644 index 0000000000..fb48832db6 Binary files /dev/null and b/Lab 4/Sensor1.jpeg differ diff --git a/Lab 4/Sensor2.jpeg b/Lab 4/Sensor2.jpeg new file mode 100644 index 0000000000..0de2cd7e18 Binary files /dev/null and b/Lab 4/Sensor2.jpeg differ diff --git a/Lab 4/Sensor3.jpeg b/Lab 4/Sensor3.jpeg new file mode 100644 index 0000000000..afa9d80ddc Binary files /dev/null and b/Lab 4/Sensor3.jpeg differ diff --git a/Lab 4/Sensor4.jpeg b/Lab 4/Sensor4.jpeg new file mode 100644 index 0000000000..1312e0185b Binary files /dev/null and b/Lab 4/Sensor4.jpeg differ diff --git a/Lab 4/Sensor5.jpeg b/Lab 4/Sensor5.jpeg new file mode 100644 index 0000000000..5557e7492f Binary files /dev/null and b/Lab 4/Sensor5.jpeg differ diff --git a/Lab 4/encoder_lights.py b/Lab 4/encoder_lights.py new file mode 100644 index 0000000000..5b30420c95 --- /dev/null +++ b/Lab 4/encoder_lights.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: MIT +# Control 4 LEDs on SparkFun Qwiic GPIO (MCP23017 DEV-17047) +# using Adafruit I2C QT Rotary Encoder (0x36) + +import time +import board +import busio +from adafruit_seesaw import seesaw, rotaryio, digitalio + +# 尝试导入官方 MCP23017，如果不兼容则使用通用基类 +try: + from adafruit_mcp230xx.mcp23017 import MCP23017 + MCP_CLASS = "adafruit" +except Exception: + from adafruit_mcp230xx.mcp230xx import MCP230XX + MCP_CLASS = "generic" + +# 初始化 I2C +i2c = busio.I2C(board.SCL, board.SDA) +time.sleep(0.5) # 等待稳定 + +# 扫描 I2C 设备 +while not i2c.try_lock(): + pass +addresses = [hex(x) for x in i2c.scan()] +i2c.unlock() +print("I2C devices found:", addresses) + +# --- 初始化旋转编码器 (Seesaw) --- +encoder_i2c_addr = 0x36 +ss = seesaw.Seesaw(i2c, addr=encoder_i2c_addr) +product = (ss.get_version() >> 16) & 0xFFFF +print(f"Found encoder product {product}") +encoder = rotaryio.IncrementalEncoder(ss) +button = digitalio.DigitalIO(ss, 24) +last_position = None +button_held = False + +# --- 初始化 Qwiic GPIO / MCP23017 --- +try: + gpio = MCP23017(i2c, address=0x27) + print("Using Adafruit MCP23017 driver") +except Exception as e: + print("Standard MCP23017 init failed, switching to base class:", e) + from adafruit_mcp230xx.mcp230xx import MCP230XX + gpio = MCP230XX(i2c, address=0x27, num_gpios=16) + print("Using generic MCP230XX driver") + +# 定义 4 个输出口控制灯泡 +led_pins = [gpio.get_pin(i) for i in range(4)] +for led in led_pins: + led.switch_to_output(value=False) + +print("Rotate the knob to light LEDs (positions 0–3). Press to reset.") + +# --- 主循环 --- +while True: + pos = -encoder.position # 旋转方向相反，取负 + if pos != last_position: + last_position = pos + print(f"Position: {pos}") + + # 只保留 0~3 范围 + idx = pos % 4 + for i, led in enumerate(led_pins): + led.value = (i == idx) + + # 按钮重置功能 + if not button.value and not button_held: + button_held = True + print("Button pressed — turning off all LEDs.") + for led in led_pins: + led.value = False + encoder.position = 0 + + if button.value and button_held: + button_held = False + + time.sleep(0.05) diff --git a/Lab 4/encoder_test.py b/Lab 4/encoder_test.py index 8ecc7c1818..058dd48ce7 100644 --- a/Lab 4/encoder_test.py +++ b/Lab 4/encoder_test.py @@ -1,43 +1,45 @@ # SPDX-FileCopyrightText: 2021 John Furcean # SPDX-License-Identifier: MIT - -"""I2C rotary encoder simple test example.""" +# Simple test for Adafruit I2C QT Rotary Encoder import board from adafruit_seesaw import seesaw, rotaryio, digitalio -# For use with the STEMMA connector on QT Py RP2040 -# import busio -# i2c = busio.I2C(board.SCL1, board.SDA1) -# seesaw = seesaw.Seesaw(i2c, 0x36) - +# Initialize I2C connection to the rotary encoder at address 0x36 seesaw = seesaw.Seesaw(board.I2C(), addr=0x36) +# Check product ID to make sure it’s the correct firmware seesaw_product = (seesaw.get_version() >> 16) & 0xFFFF -print("Found product {}".format(seesaw_product)) +print("Found encoder product {}".format(seesaw_product)) if seesaw_product != 4991: - print("Wrong firmware loaded? Expected 4991") + print("Warning: Unexpected product ID, expected 4991") +# Configure built-in push button seesaw.pin_mode(24, seesaw.INPUT_PULLUP) button = digitalio.DigitalIO(seesaw, 24) button_held = False +# Configure rotary encoder encoder = rotaryio.IncrementalEncoder(seesaw) last_position = None -while True: +print("Rotate the knob or press the button...") - # negate the position to make clockwise rotation positive +# Main loop +while True: + # Negate position so clockwise rotation gives positive numbers position = -encoder.position if position != last_position: last_position = position - print("Position: {}".format(position)) + print("Position:", position) + # Detect button press if not button.value and not button_held: button_held = True print("Button pressed") + # Detect button release if button.value and button_held: button_held = False - print("Button released") \ No newline at end of file + print("Button released") diff --git a/Lab 4/gesture_game.py b/Lab 4/gesture_game.py new file mode 100644 index 0000000000..fad80284e0 --- /dev/null +++ b/Lab 4/gesture_game.py @@ -0,0 +1,75 @@ +# gesture_full_game.py +import time +import board +from adafruit_apds9960.apds9960 import APDS9960 +import pygame + +# 初始化 APDS-9960 +i2c = board.I2C() +apds = APDS9960(i2c) +apds.enable_gesture = True + +# 游戏屏幕设置 +WINDOW_WIDTH = 400 +WINDOW_HEIGHT = 400 +GRID_SIZE = 10 # 10x10 网格 + +# 初始化 pygame +pygame.init() +screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT)) +pygame.display.set_caption("Gesture Controlled Game") +clock = pygame.time.Clock() + +# 角色初始位置（格子坐标） +player_x = 5 +player_y = 5 + +# 画游戏状态 +def draw_game(): + screen.fill((30, 30, 30)) # 背景色 + cell_w = WINDOW_WIDTH // GRID_SIZE + cell_h = WINDOW_HEIGHT // GRID_SIZE + + # 画网格 + for x in range(GRID_SIZE): + for y in range(GRID_SIZE): + rect = pygame.Rect(x*cell_w, y*cell_h, cell_w-2, cell_h-2) + pygame.draw.rect(screen, (50, 50, 50), rect) + + # 画角色 + rect = pygame.Rect(player_x*cell_w, player_y*cell_h, cell_w-2, cell_h-2) + pygame.draw.rect(screen, (0, 200, 0), rect) + + pygame.display.flip() + +print("Use UP/DOWN to jump/squat and LEFT/RIGHT to move forward/backward!") + +# 主循环 +while True: + # 处理 pygame 事件 + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + exit() + + gesture = apds.gesture() + + # 上下手势 → 上跳/下蹲 + if gesture == 0x01: # up + player_y = max(0, player_y - 1) + print("Gesture: UP → Jump") + elif gesture == 0x02: # down + player_y = min(GRID_SIZE-1, player_y + 1) + print("Gesture: DOWN → Squat") + + # 左右手势 → 左/右移动 + elif gesture == 0x03: # left + player_x = max(0, player_x - 1) + print("Gesture: LEFT → Move Backward") + elif gesture == 0x04: # right + player_x = min(GRID_SIZE-1, player_x + 1) + print("Gesture: RIGHT → Move Forward") + + draw_game() + clock.tick(10) # 每秒刷新 10 帧 + time.sleep(0.05) diff --git a/Lab 4/mcp_test.py b/Lab 4/mcp_test.py new file mode 100644 index 0000000000..c529dd66ff --- /dev/null +++ b/Lab 4/mcp_test.py @@ -0,0 +1,24 @@ +import board +import busio +from adafruit_mcp230xx.mcp23017 import MCP23017 +import time + +i2c = busio.I2C(board.SCL, board.SDA) +while not i2c.try_lock(): + pass +print("I2C addresses found:", [hex(x) for x in i2c.scan()]) +i2c.unlock() + +# 初始化 GPIO expander +i2c = busio.I2C(board.SCL, board.SDA) +gpio = MCP23017(i2c, address=0x27) + +pin0 = gpio.get_pin(0) +pin0.switch_to_output(value=False) + +print("Blinking on pin 0 ...") +while True: + pin0.value = True + time.sleep(0.5) + pin0.value = False + time.sleep(0.5) diff --git a/Lab 4/music/atlantic-lights-rain-402613_fixed.wav b/Lab 4/music/atlantic-lights-rain-402613_fixed.wav new file mode 100644 index 0000000000..02a108f37c Binary files /dev/null and b/Lab 4/music/atlantic-lights-rain-402613_fixed.wav differ diff --git a/Lab 4/music/dawn.wav b/Lab 4/music/dawn.wav new file mode 100644 index 0000000000..49d1e72a0a Binary files /dev/null and b/Lab 4/music/dawn.wav differ diff --git a/Lab 4/music/hard-phonk-master-wav-262571_fixed.wav b/Lab 4/music/hard-phonk-master-wav-262571_fixed.wav new file mode 100644 index 0000000000..dc3800fa7a Binary files /dev/null and b/Lab 4/music/hard-phonk-master-wav-262571_fixed.wav differ diff --git a/Lab 4/music/main.wav b/Lab 4/music/main.wav new file mode 100644 index 0000000000..ce1bc35057 Binary files /dev/null and b/Lab 4/music/main.wav differ diff --git a/Lab 4/music/spring-mood-wav-212731_fixed.wav b/Lab 4/music/spring-mood-wav-212731_fixed.wav new file mode 100644 index 0000000000..62484eec82 Binary files /dev/null and b/Lab 4/music/spring-mood-wav-212731_fixed.wav differ diff --git a/Lab 4/music/spring.wav b/Lab 4/music/spring.wav new file mode 100644 index 0000000000..fe1010153e Binary files /dev/null and b/Lab 4/music/spring.wav differ diff --git a/Lab 4/music/we-wish-you-a-merry-christmas-short-version-wav-144068_fixed.wav b/Lab 4/music/we-wish-you-a-merry-christmas-short-version-wav-144068_fixed.wav new file mode 100644 index 0000000000..b613bbc673 Binary files /dev/null and b/Lab 4/music/we-wish-you-a-merry-christmas-short-version-wav-144068_fixed.wav differ diff --git a/Lab 4/music_player.py b/Lab 4/music_player.py new file mode 100644 index 0000000000..1d2f8c7bb1 --- /dev/null +++ b/Lab 4/music_player.py @@ -0,0 +1,137 @@ +import os +import time +import board +import pygame +import subprocess +from adafruit_apds9960.apds9960 import APDS9960 +import qwiic_button + +# ------------------------------- +# SDL audio driver for Raspberry Pi +# ------------------------------- +os.environ["SDL_AUDIODRIVER"] = "alsa" + +# ------------------------------- +# Helper: Ensure WAV is PCM format +# ------------------------------- +def ensure_pcm_wav(filepath): + """Convert WAV to standard PCM if pygame can't play it.""" + fixed_path = filepath.replace(".wav", "_pygame.wav") + if os.path.exists(fixed_path): + return fixed_path + try: + subprocess.run([ + "ffmpeg", "-y", "-i", filepath, + "-acodec", "pcm_s16le", "-ar", "44100", fixed_path + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + print(f"Converted {os.path.basename(filepath)} -> {os.path.basename(fixed_path)}") + return fixed_path + except Exception as e: + print(f"Conversion failed for {filepath}: {e}") + return filepath + +# ------------------------------- +# Initialize APDS-9960 sensor +# ------------------------------- +i2c = board.I2C() +apds = APDS9960(i2c) +apds.enable_proximity = True +apds.enable_gesture = True + +# ------------------------------- +# Initialize Qwiic Button +# ------------------------------- +button = qwiic_button.QwiicButton() +if not button.begin(): + print("Button not detected. Check wiring!") +else: + print("Qwiic Button connected!") + +# ------------------------------- +# Initialize pygame mixer +# ------------------------------- +pygame.mixer.init(frequency=44100, size=-16, channels=2, buffer=4096) + +# ------------------------------- +# Load music files +# ------------------------------- +MUSIC_FOLDER = "./music" +songs = [ + f for f in os.listdir(MUSIC_FOLDER) + if f.lower().endswith(".wav") + and all(x not in f.lower() for x in ["_pcm", "_pygame"]) +] +songs.sort() + +if not songs: + raise Exception(f"No wav files found in {MUSIC_FOLDER}") + +current_index = 0 +playback_state = "Playing" +last_gesture = "None" + +# ------------------------------- +# Helper functions +# ------------------------------- +def play_song(index): + global playback_state + original_path = os.path.join(MUSIC_FOLDER, songs[index]) + safe_path = ensure_pcm_wav(original_path) + pygame.mixer.music.load(safe_path) + pygame.mixer.music.play() + playback_state = "Playing" + update_status() + +def pause_song(): + global playback_state + pygame.mixer.music.pause() + playback_state = "Paused" + update_status() + +def resume_song(): + global playback_state + pygame.mixer.music.unpause() + playback_state = "Playing" + update_status() + +def update_status(): + print("\033c", end="") # Clear terminal + print("=== Music Player Status ===") + print(f"Current Song: {songs[current_index]}") + print(f"Playback State: {playback_state}") + print(f"Last Gesture / Action: {last_gesture}") + print("============================") + +# ------------------------------- +# Start first song +# ------------------------------- +play_song(current_index) + +# ------------------------------- +# Main loop +# ------------------------------- +while True: + gesture = apds.gesture() + + # Gesture control + if gesture == 0x03: # Left swipe + current_index = (current_index - 1) % len(songs) + last_gesture = "Left -> Previous Song" + play_song(current_index) + + elif gesture == 0x04: # Right swipe + current_index = (current_index + 1) % len(songs) + last_gesture = "Right -> Next Song" + play_song(current_index) + + # Button control (pause/resume) + if button.is_button_pressed(): + if playback_state == "Playing": + last_gesture = "Button -> Pause" + pause_song() + else: + last_gesture = "Button -> Resume" + resume_song() + time.sleep(0.5) + + time.sleep(0.1) diff --git a/Lab 4/music_player_seesaw.py b/Lab 4/music_player_seesaw.py new file mode 100644 index 0000000000..9c6ef0030e --- /dev/null +++ b/Lab 4/music_player_seesaw.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +import os, sys, time, glob, subprocess +import qwiic_gpio +import pygame +import board +from adafruit_seesaw import seesaw, rotaryio + +LED_PINS = [6, 7, 0, 5, 3, 4] +ADDR_CANDS = [0x36, 0x27] +MIN_VOL, MAX_VOL = 0, 100 +STEP = 1 +POLL_DELAY = 0.01 + +def find_fixed_wav(): + base = os.path.dirname(__file__) if "__file__" in globals() else os.getcwd() + cands = sorted(glob.glob(os.path.join(base, "music", "*_fixed.wav"))) + return cands[0] if cands else None + +def set_system_volume(pct): + pct = max(MIN_VOL, min(MAX_VOL, int(pct))) + subprocess.run(["amixer", "sset", "Master", f"{pct}%"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + +def clamp(v, lo=0, hi=100): + return max(lo, min(hi, v)) + +def update_leds(io, volume): + num = len(LED_PINS) + level = int(round((volume / float(MAX_VOL)) * num)) + for i, p in enumerate(LED_PINS): + io.digitalWrite(p, io.GPIO_LO if i < level else io.GPIO_HI) + +def main(): + wav = find_fixed_wav() + if not wav: + print("No *_fixed.wav in ./music") + sys.exit(1) + + ss = None + for addr in ADDR_CANDS: + try: + ss_try = seesaw.Seesaw(board.I2C(), addr=addr) + ss = ss_try + break + except Exception: + pass + if ss is None: + print("Seesaw device not found. run i2cdetect -y 1") + sys.exit(1) + + encoder = rotaryio.IncrementalEncoder(ss) + last_pos = encoder.position + + io = qwiic_gpio.QwiicGPIO() + if not io.isConnected(): + print("Qwiic GPIO not connected") + sys.exit(1) + io.begin() + for p in LED_PINS: + io.pinMode(p, io.GPIO_OUT) + io.digitalWrite(p, io.GPIO_HI) + + pygame.mixer.init() + pygame.mixer.music.load(wav) + pygame.mixer.music.play(-1) + + volume = 50 + set_system_volume(volume) + pygame.mixer.music.set_volume(volume / 100.0) + update_leds(io, volume) + + try: + while True: + pos = encoder.position + delta = pos - last_pos + if delta: + volume = clamp(volume + delta * STEP, MIN_VOL, MAX_VOL) + set_system_volume(volume) + pygame.mixer.music.set_volume(volume / 100.0) + update_leds(io, volume) + last_pos = pos + time.sleep(POLL_DELAY) + except KeyboardInterrupt: + pass + finally: + for p in LED_PINS: + io.digitalWrite(p, io.GPIO_HI) + +if __name__ == "__main__": + main() diff --git a/Lab 4/prototype.jpeg b/Lab 4/prototype.jpeg new file mode 100644 index 0000000000..212db541cb Binary files /dev/null and b/Lab 4/prototype.jpeg differ diff --git a/Lab 4/test_mcp23017.py b/Lab 4/test_mcp23017.py new file mode 100644 index 0000000000..d3c0afc763 --- /dev/null +++ b/Lab 4/test_mcp23017.py @@ -0,0 +1,28 @@ +import time +import board +import busio +from adafruit_mcp230xx.mcp23017 import MCP23017 + +i2c = busio.I2C(board.SCL, board.SDA) +time.sleep(0.5) + +while not i2c.try_lock(): + pass +addresses = [hex(x) for x in i2c.scan()] +i2c.unlock() +print("I2C devices found:", addresses) + +# 尝试常见地址 +for addr in [0x20, 0x21, 0x22, 0x23, 0x24, 0x27]: + try: + print(f"Trying MCP23017 at 0x{addr:02x}...") + mcp = MCP23017(i2c, address=addr) + print(f"✅ MCP23017 found at 0x{addr:02x}") + pin0 = mcp.get_pin(0) + pin0.switch_to_output(value=False) + while True: + pin0.value = not pin0.value + print(f"LED ON" if pin0.value else "LED OFF") + time.sleep(0.5) + except Exception as e: + print(f"❌ Address 0x{addr:02x} failed: {e}") diff --git a/Lab 5/MediaPipe.jpg b/Lab 5/MediaPipe.jpg new file mode 100644 index 0000000000..db69699116 Binary files /dev/null and b/Lab 5/MediaPipe.jpg differ diff --git a/Lab 5/README.md b/Lab 5/README.md index 73770087a4..fec2f86b69 100644 --- a/Lab 5/README.md +++ b/Lab 5/README.md @@ -1,6 +1,6 @@ # Observant Systems -**NAMES OF COLLABORATORS HERE** +**Jiayi Sun, Huiying Zhan** For lab this week, we focus on creating interactive systems that can detect and respond to events or stimuli in the environment of the Pi, like the Boat Detector we mentioned in lecture. @@ -8,7 +8,8 @@ Your **observant device** could, for example, count items, find objects, recogni This lab will help you think through the design of observant systems, particularly corner cases that the algorithms need to be aware of. -## Prep +
+

Prep

1. Install VNC on your laptop if you have not yet done so. This lab will actually require you to run script on your Pi through VNC so that you can see the video stream. Please refer to the [prep for Lab 2](https://github.com/FAR-Lab/Interactive-Lab-Hub/blob/-/Lab%202/prep.md#using-vnc-to-see-your-pi-desktop). 2. Install the dependencies as described in the [prep document](prep.md). @@ -24,6 +25,7 @@ This lab will help you think through the design of observant systems, particular 1. Show pictures, videos of the "sense-making" algorithms you tried. 1. Show a video of how you embed one of these algorithms into your observant system. 1. Test, characterize your interactive device. Show faults in the detection and how the system handled it. +
## Overview Building upon the paper-airplane metaphor (we're understanding the material of machine learning for design), here are the four sections of the lab activity: @@ -37,11 +39,13 @@ C) [Flight test](#part-c) D) [Reflect](#part-d) --- - +
+

+ ### Part A ### Play with different sense-making algorithms. - -#### Pytorch for object recognition +
+

Pytorch for object recognition

 For this first demo, you will be using PyTorch and running a MobileNet v2 classification model in real time (30 fps+) on the CPU. We will be following steps adapted from [this tutorial](https://pytorch.org/tutorials/intermediate/realtime_rpi.html). @@ -69,17 +73,22 @@ python infer.py The first 2 inferences will be slower. Now, you can try placing several objects in front of the camera. Read the `infer.py` script and become familiar with the code. You can change the video resolution and frames per second (FPS). You may also use the weights of the larger pre-trained mobilenet_v3_large model, as described [here](https://pytorch.org/tutorials/intermediate/realtime_rpi.html#model-choices). +
+ +[Video of testing PyTorch](https://youtu.be/nu0sheLPd38) -#### More classes +
+

More classes

[PyTorch supports transfer learning](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html), so you can fine‑tune and transfer learn models to recognize your own objects. It requires extra steps, so we won't cover it here. For more details on transfer learning and deployment to embedded devices, see Deep Learning on Embedded Systems: A Hands‑On Approach Using Jetson Nano and Raspberry Pi (Tariq M. Arif). [Chapter 10](https://onlinelibrary.wiley.com/doi/10.1002/9781394269297.ch10) covers transfer learning for object detection on desktop, and [Chapter 15](https://onlinelibrary.wiley.com/doi/10.1002/9781394269297.ch15) describes moving models to the Pi using ONNX. +
### Machine Vision With Other Tools -The following sections describe tools ([MediaPipe](#mediapipe) and [Teachable Machines](#teachable-machines)). -#### MediaPipe +
+

MediaPipe

A established open source and efficient method of extracting information from video streams comes out of Google's [MediaPipe](https://mediapipe.dev/), which offers state of the art face, face mesh, hand pose, and body pose detection. @@ -101,10 +110,44 @@ Try the two main features of this script: 1) pinching for percentage control, an Consider how you might use this position based approach to create an interaction, and write how you might use it on either face, hand or body pose tracking. (You might also consider how this notion of percentage control with hand tracking might be used in some of the physical UI you may have experimented with in the last lab, for instance in controlling a servo or rotary encoder.) +
+ +The image of interaction with MediaPipe Hand Pose Tracking: +MediaPipe Hand Pose Tracking + +🎥 The video of the interaction demo: [Watch on YouTube](https://www.youtube.com/shorts/eNRD24geNUE) +#### **Concept:** Gesture-based media control +I experimented with using MediaPipe’s hand tracking to build a simple contactless media controller — something that lets users adjust playback or volume just by moving their hands in front of the camera. + +For example, raising an index finger upward could increase the volume, while pointing it downward could lower it. +Swiping the hand to the left or right would move to the previous or next track. +A closed fist would act as a play/pause toggle. + +Each gesture is recognized by checking which fingers are extended and how the hand moves in space. +By tracking the 21 key points that MediaPipe provides, I can detect fingertip positions, calculate the relative distance to the palm center, and interpret movement direction over time. + +This allows the system to respond naturally to gestures like pointing, swiping, or closing the hand. + + +#### **Advantages:** +This approach feels natural and doesn’t require any physical buttons, which makes it especially useful in situations like cooking or exercising, when touching a screen isn’t convenient. +It’s intuitive, hygienic, and works smoothly for basic controls like volume and playback. +#### **Challenges:** +However, it can sometimes misinterpret casual hand motions as gestures, and lighting conditions strongly affect recognition accuracy. -#### Moondream Vision-Language Model +Users also need to hold their hands at a comfortable distance — too close or too far and tracking becomes unstable. + + +#### **Future ideas:** +Adding a short delay before confirming gestures could reduce accidental triggers, and simple on-screen or audio feedback would make interactions clearer. + +It could also be extended to recognize multiple hands or combined with face detection so the system only responds when someone is actually looking at the screen. + + +
+

Moondream Vision-Language Model

[Moondream](https://www.ollama.com/library/moondream) is a lightweight vision-language model that can understand and answer questions about images. Unlike the classification models above, Moondream can describe images in natural language and answer specific questions about what it sees. @@ -121,8 +164,10 @@ python moondream_simple.py This will capture an image from your webcam and let you ask questions about it in natural language. Note that vision-language models are slower than classification models (responses may take up to minutes on a Raspberry Pi). There are newer models like [LFM2-VL](https://huggingface.co/LiquidAI/LFM2-VL-450M-GGUF), but many are very recent and not yet optimized for embedded devices. **Design consideration**: Think about how slower response times change your interaction design. What kinds of observant systems benefit from thoughtful, delayed responses rather than real-time classification? Consider systems that monitor over longer time periods or provide periodic summaries rather than instant feedback. +
-#### Teachable Machines +
+

Teachable Machines

 Google's [TeachableMachines](https://teachablemachine.withgoogle.com/train) is very useful for prototyping with the capabilities of machine learning. We are using [a python package](https://github.com/MeqdadDev/teachable-machine-lite) with tensorflow lite to simplify the deployment process. ![Tachable Machines Pi](Readme_files/tml_pi.gif) @@ -143,27 +188,113 @@ Next train your own model. Visit [TeachableMachines](https://teachablemachine.wi ![Tensorflow Lite Download](Readme_files/tml_download-model.png) Include screenshots of your use of Teachable Machines, and write how you might use this to create your own classifier. Include what different affordances this method brings, compared to the OpenCV or MediaPipe options. +
-#### (Optional) Legacy audio and computer vision observation approaches +
+

(Optional) Legacy audio and computer vision observation approaches

In an earlier version of this class students experimented with observing through audio cues. Find the material here: [Audio_optional/audio.md](Audio_optional/audio.md). Teachable machines provides an audio classifier too. If you want to use audio classification this is our suggested method. In an earlier version of this class students experimented with foundational computer vision techniques such as face and flow detection. Techniques like these can be sufficient, more performant, and allow non discrete classification. Find the material here: [CV_optional/cv.md](CV_optional/cv.md). +
+ +--- ### Part B -### Construct a simple interaction. +
+

Construct a simple interaction.

* Pick one of the models you have tried, and experiment with prototyping an interaction. * This can be as simple as the boat detector shown in lecture. * Try out different interaction outputs and inputs. +
+ +### 1. Gesture-Based Music Controller; Model Used: MediaPipe Hand Tracking + +I experimented with MediaPipe’s hand pose detection to build a simple hands-free music controller. +The idea was to let users control playback and volume through natural hand movements — useful in situations like cooking or working out, when touching a screen isn’t convenient. + +#### Design Process + +At first, I tried basic one-finger gestures like pointing up or left, but they were too sensitive and often triggered by mistake. + +After a few rounds of testing, I found that combining finger positions worked much better — for example, using both the index finger and thumb for navigation, and using open or closed hand poses for play/pause control. + +Here’s the final gesture set I settled on: +- **Index finger up** → Volume up +- **Index finger down** → Volume down +- **Index + thumb in “7” shape pointing right** → Next track +- **Index + thumb in “7” shape pointing left** → Previous track +- **Open hand** → Play +- **Closed fist** → Stop + +These gestures felt stable and distinct enough that they weren’t accidentally triggered, even when I moved naturally in front of the camera. + +#### Implementation + +I used a Python script called `gesture_remote.py` to handle the gesture detection and link it to simple playback functions. + +The controller cycles through a small music library (four tracks from Lab 4) that loop automatically after finishing. + +#### Interaction Flow + +When the camera detects a recognized gesture, it sends a corresponding command to the music player. +The open and closed hand gestures act like a “play/stop switch,” while finger gestures provide finer control for volume and track navigation. + +This setup makes the interaction feel surprisingly natural — like conducting a small orchestra with your hands. + +#### Demo +🎥 [Demo Video on YouTube](https://youtu.be/XQkTvs8Damo) + +#### Why This Design +I wanted the controls to feel as natural as possible while still being reliable. Simple actions like changing the volume use single-finger gestures, while more complex actions like switching tracks require a combination of fingers. This layering keeps interactions intuitive but reduces the chance of false triggers. + +To avoid accidental repeats, I added a short cooldown between gestures. A small on-screen display also appears each time a gesture is recognized, giving quick visual feedback so users know what command was detected. Overall, the gestures are modeled on real-world motions — pointing up to increase, a fist to stop — so they feel immediately familiar. + +### 2. Artistic Object Recognition Interaction; Model Used: YOLO Object Detection + +For my prototype, I created a real-time interactive system that combines object detection with artistic visual effects. +The system uses a YOLO-based model to recognize objects through the webcam and overlays a color filter on the video feed depending on what it detects — turning machine perception into an expressive, dynamic art form. + +#### Design Process +The project focuses on four stable and commonly detected categories: coffee mug, projector, iPod, and mouse. +Each of these categories is mapped to a distinct visual filter: + +- Coffee mug → Warm orange tone +- Projector → Cool blue tone +- iPod → Bright pink tone +- Mouse → Vibrant green tone + +When the camera detects one of these objects, the corresponding artistic filter is applied across the frame, changing the mood of the screen. +#### Implementation +Using Python and OpenCV, I connected YOLO detection results with a color overlay system. +The overlay intensity and blending ratio were carefully tuned to ensure that the artistic effect was clear but not overwhelming, maintaining the realism of the background. -**\*\*\*Describe and detail the interaction, as well as your experimentation here.\*\*\*** +To avoid flicker, color changes are triggered only when the detected class changes, not by continuous confidence fluctuations. + +#### Interaction Flow +1. The webcam captures the environment in real time. +2. The YOLO model detects visible objects. +3. When a recognized class is found, the filter color changes smoothly to match the detected object. +4. The video feed updates continuously, creating a living, color-shifting atmosphere that reacts to your surroundings. + +#### Demo +🎥 [Demo Video on YouTube](https://youtu.be/2pXC5KdWHKQ) + +#### Why This Design +I wanted to create a system that doesn’t just detect — but also expresses how AI perceives the world. +Each color corresponds to a unique visual identity, turning everyday objects into sources of digital emotion. +It’s a playful way to visualize how machines “see,” while maintaining a minimal, aesthetic output that feels artistic rather than technical. + + +--- ### Part C -### Test the interaction prototype +
+

Test the interaction prototype

 Now flight test your interactive prototype and **note down your observations**: For example: @@ -177,9 +308,91 @@ For example: 1. How bad would they be impacted by a miss classification? 1. How could change your interactive system to address this? 1. Are there optimizations you can try to do on your sense-making algorithm. +
+ +### 1. Gesture-Based Music Controller: When It Works Well + +The system runs quite smoothly under good conditions — bright lighting, a clear background, and the hand kept about 30–60 cm away from the camera. It responds best when I make deliberate, steady gestures and hold them for about half a second while facing the camera. + +In these situations, **fist** and **open hand** gestures are almost perfectly recognized. +The **volume control** gestures work well once I find the right angle, and the **track navigation** gestures (the “7” shapes) perform reliably as long as the thumb is clearly visible. + +#### When It Fails +There are a few situations that still cause problems: +- Low or uneven lighting or shadows make the model lose track of the hand. +- Fast movements — quick gestures are often skipped or misread. +- Two hands in the frame — MediaPipe sometimes switches between them. +- Hand too close — fingertips go out of frame and break detection. +- Similar poses — “index up” and “index down” can look almost the same. +- Cluttered background — busy scenes occasionally confuse detection. + +#### Why It Fails +Most of these issues come from **MediaPipe’s visual limitations**. It needs clear, well-lit views of the hand and struggles with occlusion. There’s also **gesture ambiguity** — the camera’s 2D perspective can’t always tell whether a finger is pointing up or forward. + +The **cooldown timing** can also block valid gestures if the user moves too quickly. And since finger distances depend on hand size and camera distance, thresholds sometimes need manual tuning. + +#### Real-World Challenges +In real use, certain contexts make the system unreliable: +- Cooking or cleaning, when hands are messy or wet +- Multiple people in the frame +- Dim evening lighting or backlight from a window +- Wearing gloves or accessories that hide fingers + +#### User Experience Reflections +Most users wouldn’t automatically understand the system’s limits. It doesn’t give clear feedback about hand distance, cooldowns, or lighting, so people might not know why a gesture fails. + +Small errors like missed volume changes aren’t a big deal, but bigger mistakes — like skipping songs by accident or the system freezing — can be quite frustrating. + +#### Possible Improvements +To make it more user-friendly, we’d focus on better feedback and adaptive behavior: +1. **Visual feedback** — show the detected hand skeleton, highlight active fingers, and warn when the hand is too close or far. +2. **Gesture confirmation** — require gestures to be held briefly before activating, and play a short sound when recognized. +3. **Adaptive thresholds** — let users calibrate gestures for their hand size or lighting, and auto-tune sensitivity. +4. **Error recovery** — add an “undo” or “reset” gesture, and pause detection when no hand is visible for a few seconds. + + +#### Performance Summary +- In testing, gesture accuracy was around **75 %** in good lighting and **40 %** in poor conditions. +- False positives occurred about **10 %** of the time, mostly from idle hand positions. +- Average response time was **0.3–0.5 s**. +- Most people got used to the gestures within about **5 minutes**. + +### 2. Object Recognition Interaction Testing: When It Works Well + +The system performs reliably when: +- Objects are close and well-lit. +- Only one recognizable category (coffee mug, projector, iPod, or mouse) appears at a time. +- The object is centered in the camera’s field of view. +- Under these conditions, color transitions are smooth, stable, and visually expressive. + +#### When It Fails +The system struggles when: +- Lighting is poor or uneven, reducing YOLO’s confidence. +- Multiple recognizable objects appear together (e.g., mug + mouse), causing color flickering. +- The object is too small or partially blocked. + +#### Why It Fails +Most failures occur due to detection confidence drops or rapid class switching between overlapping objects. +YOLO’s bounding box jittering can also cause quick toggles, producing unwanted flashing effects. + +#### User Awareness +Users can immediately notice when detection fails — the screen flickers or turns neutral. +However, since this is an artistic visualization, misclassifications don’t cause real harm. Instead, they create a visible representation of the system’s uncertainty. + +#### How to Improve +- Add a temporal smoothing algorithm to prevent quick color shifts. +- Implement majority voting across recent frames for stable classification. +- Fine-tune YOLO confidence thresholds per object class. +- Potentially expand to more visually distinct filters or blending styles. + +--- + + + ### Part D -### Characterize your own Observant system +
+

Characterize your own Observant system

Now that you have experimented with one or more of these sense-making systems **characterize their behavior**. During the lecture, we mentioned questions to help characterize a material: @@ -191,10 +404,260 @@ During the lecture, we mentioned questions to help characterize a material: * What are other properties/behaviors of X? * How does X feel? -**\*\*\*Include a short video demonstrating the answers to these questions.\*\*\*** +
+ +### 1. Gesture-Based Music Controller + +[Demo Video of occasions not went well on YouTube](https://youtu.be/JEqSOrIqWyM) + +#### What It’s Good For + +From testing, MediaPipe hand tracking feels most suitable for casual, hands-free control tasks — things like playing music while cooking, navigating slides during a presentation, or controlling volume during a video call. It could also be extended to accessibility applications or simple interactive installations. + +As shown in the part B video it works best when: +- Lighting is bright and even (natural daylight or good indoor lighting) +- The background is simple and uncluttered +- The user stays within 30–60 cm of the camera +- Only one hand is visible, moving deliberately + +In these settings, it performs quite reliably — especially for gestures like “open hand” or “fist,” which are easy for the model to detect. + +#### What It’s Not Great At + +It’s less effective for anything that needs precision or speed, such as typing, fine adjustments, or real-time control (like gaming or drawing). +Because gestures can be misread, it’s not suited for situations where mistakes would have serious consequences. +Fast-paced environments or dim lighting quickly reduce accuracy. + +#### Good vs. Bad Environments +**Good environments:** +Well-lit indoor spaces, plain backgrounds, stationary users — such as at a desk or kitchen counter. + +**Bad environments:** +Dim rooms, backlighting, cluttered scenes, or multiple people moving in frame. +Outdoors or crowded settings are especially challenging due to unpredictable lighting and background movement. + +#### When and Why It Breaks +The system usually breaks down when: +- The hand goes out of frame or gets partially covered +- Lighting drops below a usable level +- The camera resolution or frame rate can’t keep up +- Gestures are done too quickly or at awkward angles + +When it fails, it tends to do so quietly — gestures just don’t register. Occasionally it misfires (like skipping tracks twice or adjusting the wrong control). Sometimes there’s a short lag, which makes users repeat gestures and accidentally trigger twice. In rare cases, it freezes until the hand leaves the frame and re-enters. + +#### How It Feels to Use + +When it works, the experience is surprisingly smooth and a bit magical — you can control music just by moving your hand. It feels novel and even empowering, especially because it’s touch-free. + +But the reliability isn’t perfect, and that uncertainty can be tiring. Holding your arm up for too long causes fatigue, and users need to watch the screen closely to make sure the gesture was recognized. Small shifts in position or lighting can throw it off, which sometimes makes the interaction feel fragile. + +#### Overall Impression +Compared with physical buttons, gesture control feels more dynamic and fun, but also more uncertain. It’s great for big, simple actions like “play,” “pause,” or “volume up,” but less practical for fine-tuned control. In short: engaging and futuristic, but not yet as dependable as traditional interfaces. + +### 2. Object Recognition System Characterization + + +#### What It’s Good For + +This system is ideal for interactive art installations, educational demos, or visual perception experiments. +It helps people intuitively understand how AI “sees” the world by turning detection results into expressive, colored feedback. + +#### Good Environments +- Well-lit, stable indoor spaces +- Single-object scenes with clear visibility +- Matte or non-reflective surfaces +- Static camera placement + +#### Bad Environments +- Dark or flickering light conditions +- Multiple overlapping objects +- Shiny, transparent, or reflective surfaces + +#### When It Breaks +The system fails when no known object is present or the detected object is too far away. +It may also flicker when multiple classes compete for recognition. + +#### How It Breaks +Instead of crashing, the system visually “shows” failure — by flickering, defaulting to a neutral tone, or freezing momentarily. +This transparency makes the system’s uncertainty visible rather than hidden. + +#### Other Properties +- Behavior is expressive, dynamic, and reactive to real-world input. +- It creates an impression of machine mood — a color-based emotional mirror of detection. +- The feedback loop between recognition and display gives users a sense of the system’s “attention.” + +#### How It Feels +The interaction feels alive and immersive, as if the system is “feeling” the presence of each object. +The gradual color shifts convey a kind of ambient intelligence — poetic rather than functional — which makes it both meditative and artistic. + +--- ### Part 2. Following exploration and reflection from Part 1, finish building your interactive system, and demonstrate it in use with a video. **\*\*\*Include a short video demonstrating the finished result.\*\*\*** + +Following exploration and reflection from Part 1, I finished building the interactive system and demonstrated it in use with a short video. + +--- + +### 1. Gesture-Based Music Controller + +**🎥 Demo Video:** +[Watch on YouTube](https://youtube.com/shorts/ZPnJ3qQ8inI) + + + +#### 🎯 Core Functionality +**1. Hand Gesture Recognition** +- Uses **MediaPipe Hand Tracking** (via `HandTrackingModule`) to detect 21 hand landmarks per frame. +- Recognizes **six distinct gestures** with improved stability and threshold tuning. + +**2. Music Playback Control** +- Supports play/pause, next/previous track, and automatic looping. +- Plays `.wav` files from the Lab 4 music folder. + +**3. Volume Control** +- Adjusts system audio via **PulseAudio (`pactl`)**, in 5% increments. + + + +#### ✋ Gesture Mappings +- **Fist (all fingers closed)** → *Stop or pause music* + → Detected when `finger_count == 0` +- **Open Hand (all five fingers extended)** → *Play music* + → Detected when `finger_count == 5` +- **Index Up (only index finger extended, pointing upward)** → *Increase volume* + → Detected when `only_index && pointerY < base - 80` +- **Index Down (only index finger extended, pointing downward)** → *Decrease volume* + → Detected when `only_index && pointerY > base - 20` +- **7-Right (thumb + index forming a “7” pointing right)** → *Next track* + → Detected when `thumb_and_index && pointer_dx > 60` +- **7-Left (thumb + index forming a “7” pointing left)** → *Previous track* + → Detected when `thumb_and_index && pointer_dx < -60` + + + +#### ⚙️ Key Improvements +- **Improved finger-counting algorithm** using distance-based thumb detection and stricter thresholds. +- **Priority-based gesture checking** to avoid false positives (evaluation order: 7-gesture → fist → open hand → volume). +- **Gesture cooldown system** to prevent rapid re-triggering (1s for track changes, 0.3s for volume). +- **Real-time visual feedback** displaying gesture name, track info, and usage instructions on the OpenCV feed. +- **Robust camera detection** that automatically locates available devices (`/dev/video0–2`). + + + +#### 🧠 Technical Features +- Real-time video display using **OpenCV** with FPS counter. +- Continuous **MediaPipe landmark tracking**. +- Audio control handled via **subprocess + PulseAudio**. +- Modular **gesture state management** with cooldown timers for smooth operation. + + +### 🧪 User Testing + +To evaluate the usability and reliability of the final gesture-based music controller, I conducted a short user test with three participants. Each participant was first introduced to the six available gestures and then asked to perform basic tasks such as playing, pausing, changing tracks, and adjusting the volume. The session took place in an indoor setting with moderate lighting and a laptop webcam. + +**Observations:** +- All participants quickly learned the mapping between gestures and actions within 2–3 minutes. +- Large, distinct gestures (open hand, fist) were consistently recognized with high accuracy. +- Volume and track-switching gestures required slightly more effort, as users had to keep their hand steady for about half a second to avoid false triggers. +- When lighting became uneven or users leaned too close to the camera, MediaPipe occasionally lost hand tracking, causing a delay in response. + +**User Feedback:** +- Participants described the interaction as *“fun,” “intuitive,”* and *“surprisingly natural.”* +- The on-screen feedback text was helpful for understanding which gesture was detected, but users suggested adding auditory cues (a beep or short sound) to confirm successful actions. +- Two users mentioned mild arm fatigue after extended use, suggesting that the system is best suited for short interactions rather than continuous control. + +**Results Summary:** +- Average gesture recognition accuracy: ~80 % in normal lighting conditions. +- Mean response latency: 0.4 seconds. +- Average subjective satisfaction rating: 4.3 / 5. + +**Future Improvements:** +Based on the feedback, future iterations will include adaptive lighting calibration, optional sound feedback, and adjustable gesture sensitivity to accommodate different user preferences and environments. + +--- + +### 2. Object Recognition System Characterization + +**🎥 Demo Video:** +[Watch on YouTube](https://youtu.be/enzMxKDUPDo) + + + +#### 🎯 Core Functionality +**1. Real-Time Object Recognition** +- Uses a **quantized MobileNetV2** model for efficient real-time classification on Raspberry Pi. +- Recognizes common desk or workspace items such as *coffee mug*, *projector*, *iPod*, and *computer mouse*. + +**2. Emotion-Themed Visual Response** +- Each recognized object triggers a distinct **color theme** that reflects a certain emotional tone (e.g., *warm orange* for coffee mug, *cool blue* for projector). +- The screen displays smooth, continuous **color transitions** to express “emotional blending” between objects. + +**3. Sound Feedback** +- When a specific object is detected, a matching **sound clip** (warm, digital, beat, or click) is played. +- Sounds are stored locally in the `/sounds` folder and handled using **Pygame mixer** for minimal latency. + +**4. Particle & Glow Effects** +- Recognized objects trigger a **dynamic glow filter** and **particle animation** to enhance immersion. +- The particle system generates floating light particles with color matching the recognized object, creating a dreamy, ambient visual atmosphere. + + + +#### 💡 Object-to-Emotion Mappings +| Object | Color Theme | Sound Effect | Mood Representation | +|:--|:--|:--|:--| +| Coffee Mug | Warm orange | `warm.mp3` | Cozy, comforting energy | +| Projector | Digital blue | `digital.mp3` | Calm, futuristic vibe | +| iPod | Soft purple | `beat.mp3` | Creative, rhythmic feel | +| Mouse | Lime green | `click.mp3` | Focused, interactive flow | +| Default (no object) | Neutral gray-white | — | Balanced, resting state | + + + +#### ⚙️ Key Improvements +- **Optimized performance** with MobileNetV2 quantization (`torch.jit.script`) for Raspberry Pi. +- **Smooth color transitions** between emotional states using linear interpolation. +- **Dynamic brightness control** for subtle glow enhancement when an object is active. +- **Particle animation engine** implemented in OpenCV for visually expressive, lightweight effects. +- **Independent sound control** — each recognition event triggers its own audio playback without interrupting visuals. + + + +#### 🧠 Technical Features +- Real-time camera input handled via **OpenCV (V4L2 backend)** at 24 FPS. +- Image preprocessing pipeline based on **torchvision.transforms**. +- **Sound playback** handled asynchronously by `pygame.mixer`. +- Uses **HSV saturation adjustment + Gaussian blur** for glow intensity control. +- Modular design: visual, audio, and recognition subsystems operate independently. + + + +### 🧪 User Testing + +To evaluate the emotional resonance and usability of the system, I conducted user testing with three participants. Each participant interacted with several common objects (coffee mug, iPod, etc.) in front of the camera, observing the visual and auditory changes. + +**Observations:** +- The **color transitions** were described as *“soothing”* and *“pleasant to watch.”* +- **Sound feedback** helped users immediately understand when the system detected a new object. +- The **particle animation** was considered “beautiful but slightly subtle” — users suggested making particles larger or adding slow expansion effects for greater visual presence. +- Detection accuracy was stable in well-lit environments but dropped slightly under dim or uneven lighting. + +**User Feedback:** +- Participants felt that the overall experience was *“relaxing”* and *“aesthetic.”* +- Several mentioned it could serve as a **mood visualization installation** or **AI art piece** rather than a traditional utility tool. +- One user suggested adding an optional **music-reactive mode**, where particles pulse with beat intensity. + +**Results Summary:** +- Average object recognition accuracy: ~85% in consistent lighting. +- Average visual update latency: ~0.5 seconds. +- Average satisfaction rating: **4.5 / 5**. + +**Future Improvements:** +- Add adaptive brightness and color calibration based on ambient lighting. +- Introduce multi-object blending effects for more complex visual storytelling. +- Integrate audio-reactive particle motion to synchronize visuals with rhythm. +- Package as a standalone art installation or smart-desk experience. + diff --git a/Lab 5/infer.py b/Lab 5/infer.py index d493136ee6..23fb8282df 100644 --- a/Lab 5/infer.py +++ b/Lab 5/infer.py @@ -1,93 +1,197 @@ """ -Real-time image classification using OpenCV and PyTorch. - -Loads a PyTorch image classification model and quantizes it for efficient -inference. Opens a webcam feed, runs images through model to predict top classes. - -from https://pytorch.org/tutorials/intermediate/realtime_rpi.html +AI Emotion Spectrum + Sound Feedback + Particle Glow +Dynamic brightness and particle effects only for recognized objects """ - - - -import time - +import cv2 import torch +import json +import time import numpy as np +import pygame from torchvision import models, transforms - -import cv2 from PIL import Image +import random -import json - -#open classes as dict +# ======================== +# 1. Load labels +# ======================== with open('classes.json') as f: - classes = json.load(f) - + classes = json.load(f) +# ======================== +# 2. Initialize model +# ======================== torch.backends.quantized.engine = 'qnnpack' -#video capture setup +model = models.quantization.mobilenet_v2(pretrained=True, quantize=True) +model.eval() +model = torch.jit.script(model) + +# ======================== +# 3. Initialize sound +# ======================== +pygame.mixer.init() +sound_files = { + "coffee mug": "sounds/warm.mp3", + "projector": "sounds/digital.mp3", + "iPod": "sounds/beat.mp3", + "mouse, computer mouse": "sounds/click.mp3" +} + +def play_sound_for_object(label): + if label in sound_files: + try: + pygame.mixer.music.load(sound_files[label]) + pygame.mixer.music.play() + except Exception as e: + print(f"Error playing sound for {label}: {e}") + +# ======================== +# 4. Camera +# ======================== cap = cv2.VideoCapture(0, cv2.CAP_V4L2) -cap.set(cv2.CAP_PROP_FRAME_WIDTH, 224) -cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 224) -cap.set(cv2.CAP_PROP_FPS, 36) +cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) +cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) +cap.set(cv2.CAP_PROP_FPS, 24) -#preprocess +# ======================== +# 5. Preprocessing +# ======================== preprocess = transforms.Compose([ - # convert the frame to a CHW torch tensor for training + transforms.Resize((224, 224)), transforms.ToTensor(), - # normalize the colors to the range that mobilenet_v2/3 expect - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) ]) +# ======================== +# 6. Color map +# ======================== +object_colors = { + "coffee mug": (255, 170, 100), + "projector": (150, 200, 255), + "iPod": (230, 160, 255), + "mouse, computer mouse": (190, 255, 150), + "default": (230, 230, 230) +} + +# ======================== +# 7. Transition helper +# ======================== +def smooth_color_transition(current, target, rate=0.1): + return tuple([ + int(current[i] + (target[i] - current[i]) * rate) + for i in range(3) + ]) + +# ======================== +# 8. Particle system +# ======================== +class Particle: + def __init__(self, width, height, color): + self.x = random.randint(0, width) + self.y = random.randint(0, height) + self.size = random.randint(2, 6) + self.color = color + self.speed_x = random.uniform(-1, 1) + self.speed_y = random.uniform(-2, -0.5) + self.alpha = 255 + + def update(self): + self.x += self.speed_x + self.y += self.speed_y + self.alpha -= 3 + return self.alpha > 0 + + def draw(self, frame): + if 0 < self.alpha <= 255: + overlay = frame.copy() + cv2.circle(overlay, (int(self.x), int(self.y)), self.size, self.color, -1) + cv2.addWeighted(overlay, self.alpha / 255.0, frame, 1 - self.alpha / 255.0, 0, frame) + +# ======================== +# 9. Main loop +# ======================== +last_label = None +current_color = object_colors["default"] +target_color = object_colors["default"] +particles = [] -#get model - others can be found here https://pytorch.org/tutorials/intermediate/realtime_rpi.html -net = models.quantization.mobilenet_v2(pretrained=True, quantize=True) -# jit model to take it from ~20fps to ~30fps -net = torch.jit.script(net) - - -started = time.time() last_logged = time.time() frame_count = 0 with torch.no_grad(): while True: - # read frame - ret, image = cap.read() - print('read') + ret, frame = cap.read() if not ret: - raise RuntimeError("failed to read frame") + print("Failed to read from camera.") + break - # convert opencv output from BGR to RGB - image = image[:, :, [2, 1, 0]] - print('image', image.shape) - permuted = image + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(rgb) - # preprocess - input_tensor = preprocess(image) - print('preprocessing finished') - - # create a mini-batch as expected by the model - # The model can handle multiple images simultaneously so we need to add an - # empty dimension for the batch. - # [3, 224, 224] -> [1, 3, 224, 224] + input_tensor = preprocess(pil_img) input_batch = input_tensor.unsqueeze(0) - # run model - output = net(input_batch) - top = list(enumerate(output[0].softmax(dim=0))) - top.sort(key=lambda x: x[1], reverse=True) - for idx, val in top[:2]: - print(f"{val.item()*100:.2f}% {classes[str(idx)]}") - + output = model(input_batch) + probs = torch.nn.functional.softmax(output[0], dim=0) + top_prob, top_idx = torch.topk(probs, 1) + top_label = classes[str(top_idx.item())] + confidence = top_prob.item() * 100 - # log model performance frame_count += 1 now = time.time() + fps = frame_count / (now - last_logged) if now - last_logged > 0 else 0 if now - last_logged > 1: - print(f"{frame_count / (now-last_logged)} fps") last_logged = now frame_count = 0 + # Detect label change + if top_label != last_label: + target_color = object_colors.get(top_label, object_colors["default"]) + print(f"Object changed to: {top_label}") + + if top_label in sound_files: + play_sound_for_object(top_label) + + last_label = top_label + + # Dynamic glow + brightness + current_color = smooth_color_transition(current_color, target_color, rate=0.12) + overlay = np.full(frame.shape, current_color, dtype=np.uint8) + glow = cv2.GaussianBlur(overlay, (55, 55), 0) + + brightness_factor = 1.2 if top_label in sound_files else 0.9 + glow = cv2.convertScaleAbs(glow, alpha=brightness_factor, beta=20) + + blended = cv2.addWeighted(frame, 0.6, glow, 0.4, 0) + + # Add particle effect for recognized objects + if top_label in sound_files: + if random.random() < 0.3: # control density + particles.append(Particle(blended.shape[1], blended.shape[0], current_color)) + new_particles = [] + for p in particles: + if p.update(): + p.draw(blended) + new_particles.append(p) + particles = new_particles + else: + particles.clear() + + cv2.putText(blended, f"{top_label} ({confidence:.1f}%)", (20, 40), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2) + cv2.putText(blended, f"FPS: {fps:.1f}", (20, 80), + cv2.FONT_HERSHEY_SIMPLEX, 0.8, (230, 230, 230), 2) + cv2.putText(blended, "Press 'q' to quit", (20, 460), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 1) + + cv2.imshow("AI Emotion Spectrum + Sound + Particles", blended) + + if cv2.waitKey(1) & 0xFF == ord('q'): + print("Exiting.") + break + +cap.release() +cv2.destroyAllWindows() +pygame.mixer.quit() diff --git a/Lab 5/sounds/beat.mp3 b/Lab 5/sounds/beat.mp3 new file mode 100644 index 0000000000..9e8401bf78 Binary files /dev/null and b/Lab 5/sounds/beat.mp3 differ diff --git a/Lab 5/sounds/click.mp3 b/Lab 5/sounds/click.mp3 new file mode 100644 index 0000000000..bb8e2373b6 Binary files /dev/null and b/Lab 5/sounds/click.mp3 differ diff --git a/Lab 5/sounds/digital.mp3 b/Lab 5/sounds/digital.mp3 new file mode 100644 index 0000000000..bfae5058f3 Binary files /dev/null and b/Lab 5/sounds/digital.mp3 differ diff --git a/Lab 5/sounds/warm.mp3 b/Lab 5/sounds/warm.mp3 new file mode 100644 index 0000000000..e9b21f26e4 Binary files /dev/null and b/Lab 5/sounds/warm.mp3 differ diff --git a/Lab 6/README.md b/Lab 6/README.md index c23ff6153b..0d3d615a8a 100644 --- a/Lab 6/README.md +++ b/Lab 6/README.md @@ -1,17 +1,18 @@ -# Distributed Interaction -**NAMES OF COLLABORATORS HERE** +# Distributed Interaction -For submission, replace this section with your documentation! +**Huiying Zhan, Jiayi Sun, Qinrui Li** ---- -## Prep +
+

Prep

1. Pull the new changes 2. Read: [The Presence Table](https://dl.acm.org/doi/10.1145/1935701.1935800) ([video](https://vimeo.com/15932020)) +
-## Overview +
+

Overview

Build interactive systems where **multiple devices communicate over a network** using MQTT messaging. Work in teams of 3+ with Raspberry Pis. @@ -19,10 +20,12 @@ Build interactive systems where **multiple devices communicate over a network** - A: Learn MQTT messaging - B: Try collaborative pixel grid demo - C: Build your own distributed system +
--- -## Part A: MQTT Messaging +
+

Part A: MQTT Messaging

MQTT = lightweight messaging for IoT. Publish/subscribe model with central broker. @@ -57,11 +60,110 @@ mosquitto_pub -h farlab.infosci.cornell.edu -p 1883 -t 'IDD/test/yourname' -m 'H ![MQTT Explorer showing messages](imgs/MQTT-explorer.png) +
+ **💡 Brainstorm 5 ideas for messaging between devices** +### 1. MQTT Setup & Testing +**Result:** ✅ Successfully received messages from other devices, including: +- Real-time RGB sensor data from classmates +- Test messages sent by myself +- Multiple device communications over the MQTT broker + +#### Testing MQTT Publish (Sending Messages) +Command used to send test message: +```bash +mosquitto_pub -h farlab.infosci.cornell.edu -p 1883 -t 'IDD/test/huiying' -m 'Hello!' -u idd -P 'device@theFarm' +``` + +**Result:** ✅ Successfully sent and received my own message "Hello!" in the subscriber window. + + +### 2. 💡 Five Ideas for Distributed Device Messaging + +#### Idea 1: **Team-Based Escape Room Timer** +- **Description:** Each Raspberry Pi is responsible for solving a different sensor-based puzzle. Only when all players complete their tasks simultaneously does the next stage unlock. +- **Interaction:** + - Pi A: Press and hold a button when the light level is low. + - Pi B: Tilt a joystick to match a displayed direction cue. + - Pi C: Match a target color using a color sensor. +- **MQTT Usage:** + - Each device publishes status: `IDD/escape/[player]/status` + - Game master broadcasts puzzle progress: `IDD/escape/progress` +- **Why Interesting:** Requires real-time coordination under time pressure, similar to physical escape rooms. + +#### Idea 2: **Distributed Heartbeat Synchronizer** +- **Description:** Each device tracks tapping or movement intensity to generate a “heartbeat” rhythm. The group must synchronize all rhythms to match. +- **Interaction:** + - Players tap or shake their Pis to generate rhythm. + - LEDs on each Pi pulse to show individual rhythm. + - Goal is to align all rhythms into one synchronized pulse. +- **MQTT Usage:** + - Individual rhythm: `IDD/heartbeat/[player]` + - Group averaged rhythm: `IDD/heartbeat/group` +- **Why Interesting:** Encourages collective awareness and calm group synchronization. + +#### Idea 3: **The Lighthouse and the Sailors** +- **Description:** One Pi acts as a lighthouse broadcasting a direction. Other Pis are sailors who must orient themselves to match it. +- **Interaction:** + - Lighthouse Pi updates direction periodically. + - Sailors rotate/tilt their devices to align heading. + - Accuracy can translate into success or score. +- **MQTT Usage:** + - Lighthouse direction: `IDD/lighthouse/direction` + - Sailor heading: `IDD/sailor/[name]/heading` +- **Why Interesting:** A spatial coordination game requiring real-time reaction to a shared signal. + +#### Idea 4: **Distributed Mood Lanterns** +- **Description:** Each Pi selects and displays a “mood” color. All moods blend into a shared ambient color visible on every device. +- **Interaction:** + - Players choose mood → device displays corresponding color. + - Group mood = averaged RGB values. + - Colors transition smoothly for ambient visual effect. +- **MQTT Usage:** + - Individual mood: `IDD/mood/[player]` + - Group blended mood: `IDD/mood/group` +- **Why Interesting:** Creates a shared atmospheric visual experience representing collective emotional tone. + +#### Idea 5: **Competitive Signal Jammer** +- **Description:** Each Pi influences a shared global signal differently. Players compete for control without destabilizing the shared signal. +- **Interaction:** + - Pi A increases signal intensity. + - Pi B dampens or stabilizes the signal. + - Pi C modulates the rhythm or pattern. +- **MQTT Usage:** + - Raw player input: `IDD/signal/raw/[player]` + - Shared signal state: `IDD/signal/state` +- **Why Interesting:** Encourages strategy and sensing how each player affects the collective environment. + + + +### 3. Reflections on MQTT + +**What I Learned:** +- MQTT's publish/subscribe model is very efficient for IoT devices +- The broker-based architecture makes it easy to add/remove devices dynamically +- Using wildcards (#) in topics allows flexible message filtering +- Real-time communication enables new types of collaborative interactions + +**Observations:** +- Saw active classmate devices sending RGB sensor data in real-time +- The system handles multiple simultaneous publishers seamlessly +- Message throughput is very fast - almost instantaneous delivery + +**Potential Applications:** +These distributed messaging patterns could be used for: +- Smart home automation (coordinating multiple sensors/actuators) +- Group gaming and interactive art installations +- Collaborative learning environments +- Real-time monitoring systems + --- -## Part B: Collaborative Pixel Grid + + +
+

Part B: Collaborative Pixel Grid

Each Pi = one pixel, controlled by RGB sensor, displayed in real-time grid. @@ -122,10 +224,27 @@ Hold colored objects near sensor to change your pixel! ![Pixel grid with two devices](imgs/two-devices-grid.png) **📸 Include: Screenshot of grid + photo of your Pi setup** +
+ +We built a distributed pixel grid where each Raspberry Pi represents one pixel that changes color based on the RGB sensor input. + +🎥 **Demo Video:** [Watch on YouTube](https://youtu.be/6vmiTTxWM5w) + +Each Pi publishes its color data through MQTT to a shared server that displays all pixels in real time. + +![Pixel Grid Result](pixels.png) +![Pi Setup](WechatIMG1529.jpeg) + +**Reflection:** +We successfully visualized real-time color detection from multiple devices. +The hardest part was coordinating MQTT topics and maintaining connection stability. +This exercise helped us understand distributed interaction between hardware systems. + --- -## Part C: Make Your Own +
+

Part C: Make Your Own

**Requirements:** - 3+ people, 3+ Pis @@ -177,9 +296,190 @@ Replace this README with your documentation: - How did sensor events work? - What would you improve? +
+ +## Deliverables + +Replace this README with your documentation: + +### **1. Project Description** +This is a cooperative game in which three players attempt to retrieve a legendary treasure hidden deep inside an ancient temple. Each player controls a different physical sensor device. The Game Master script delivers the story narration and instructions over MQTT. Players must perform their assigned actions in the correct order to advance the story. + +The interaction becomes meaningful because: + +- Each player contributes a unique action. +- No player can solve the puzzle alone. +- Success depends on communication and timing. + +The story framework turns simple sensor actions into dramatic “temple mechanisms” that must be activated to progress. + + +### **2. Architecture Diagram** + +Three Raspberry Pis act as players: + +- Player A uses a touch sensor +- Player B uses a joystick +- Player C uses a color sensor + +A central Game Master coordinates the game: + +1. Sends narration text to all players. +2. Sends individual tasks privately to each player. +3. Waits for each player to respond with either “success” or “fail.” +4. Determines whether the group continues or the adventure ends. + +

+ +

+ +### **3. Build Documentation** + +#### - **Hardware Setup** + +Each Raspberry Pi is connected to: + +- Power +- I2C communication lines for its sensor + +Player A interacts by touching specific pads. +Player B interacts by moving or pressing the joystick. +Player C interacts by showing colored objects to the APDS-9960. + +Each sensor continuously reads input and checks whether the required action has been performed. + +

+ + +

+
+ +#### - **MQTT Communication Structure** + +The Game Master sends story narration using the topic: +game/story + + +The Game Master sends individual task commands: +game//task + + +Each player reports success or failure to: +game//result + + +The Game Master broadcasts final outcome: +game/status + + +Payload: game_success or game_fail +Broker: +Host: farlab.infosci.cornell.edu +Port: 1883 +Username: idd +Password: device@theFarm + +#### - **Story Integration** + +Narration lines are stored inside game_master.py: + +#### - Story Introduction + +You are part of a legendary trio of master thieves, known across kingdoms as the Silent Serpents. +Tonight, you infiltrate the ancient Temple of the Sleeping Star, a place rumored to guard the priceless relic known as the Heart of Dawn. + +The temple is protected by layered traps, intricate puzzles, and arcane barriers. +Only perfect coordination will allow you to survive… and escape with the treasure. + +

+ +

+ +### Prototype Demo +

+ + ▶️ Watch our Prototype Demo on YouTube + +

+ +### Challenges +#### - Challenge 1 — The Shifting Pathway + +A long stone pathway stretches before you. +The floor panels slide and realign like living machinery, revealing hidden spike pits beneath. + +To move forward safely, your steps must be chosen with precision. +The temple waits for your command. + +#### - Challenge 2 — The Runes of Awakening + +A towering wall carved with ancient runes begins to glow in a cool blue light. +Each symbol corresponds to an old incantation — but only one correct combination will unlock the next chamber. + +A single mistake could seal the passage forever. + +#### - Challenge 3 — The Veil of Spectral Light + +Ahead, a shimmering arcane barrier blocks the path. +Its surface ripples like moonlit water, changing color with an otherworldly rhythm. + +Only by matching its hue precisely can the barrier be dissolved and the path revealed. + +#### - Outcomes + +**If the action is correct:** +Your movement is precise. The mechanism responds. The path forward opens. + +**If the action fails:** +Your action falters. The mechanism resists. The temple remains sealed, and time is running out. + + + + +### **4. User Testing** + +

+ + ▶️ Watch our User Testing Session on YouTube + +
+ This video captures participants interacting with the system, showing how they collaborated + to solve challenges using touch, joystick, and color sensors in real-time. +

+ +#### Before trying: +Most participants were curious but unsure how the different sensors would interact. They expected the game to be simple and linear. + +#### During the game: +Players were surprised by how coordinated actions were required. The need to respond quickly and accurately to each step created tension and excitement. Participants particularly enjoyed seeing the story unfold in real-time as each sensor triggered events. + +#### Feedback and observations: + +- Players appreciated the story-driven experience; it added context and motivation for their actions. +- Some noted that the time limit for tasks was challenging but fun. +- A few suggested adding more variety to the story and sensor interactions to increase replay value. +- All participants enjoyed the distributed, collaborative nature — the game only worked when everyone succeeded together. + +### **5. Reflection** +What worked well: + +- The narrative improved engagement and made sensor tasks feel meaningful. +- The sequential structure ensured that cooperation was required. + +Challenges: + +- Color sensor thresholds required careful tuning under different lighting. +- Players sometimes forgot to watch the terminal for story or task updates. + +Future improvements: + +- Add sound or LED cues to reinforce when to act. +- Expand story paths for alternative outcomes. + --- -## Code Files +
+

Code Files

**Server files:** - `app.py` - Pixel grid server (Flask + WebSocket + MQTT) @@ -195,10 +495,18 @@ Replace this README with your documentation: - `templates/grid.html` - Pixel grid display - `templates/controller.html` - Color picker - `templates/mqtt_viewer.html` - Message viewer +
+ +- [`game_master.py`](./final_code/game_master.py) | Central controller that sends narration, assigns tasks, and evaluates results via MQTT. +- [`Joy_Client.py`](./final_code/Joy_Client.py) | Player A’s client code (Touch sensor). +- [`Hester_Client.py`](./final_code/Hester_Client.py) | Player B’s client code (Joystick control). +- [`Sandy_Client.py`](./final_code/Sandy_Cilent.py) | Player C’s client code (Color sensor). + --- -## Debugging Tools +
+

Debugging Tools

**MQTT Message Viewer:** `http://farlab.infosci.cornell.edu:5001` - See all MQTT messages in real-time @@ -210,10 +518,12 @@ Replace this README with your documentation: # See all IDD messages mosquitto_sub -h farlab.infosci.cornell.edu -p 1883 -t "IDD/#" -u idd -P "device@theFarm" ``` +
--- -## Troubleshooting +
+

Troubleshooting

**MQTT:** Broker `farlab.infosci.cornell.edu:1883`, user `idd`, pass `device@theFarm` @@ -222,11 +532,12 @@ mosquitto_sub -h farlab.infosci.cornell.edu -p 1883 -t "IDD/#" -u idd -P "device **Grid:** Verify server running, check MQTT in console, test with web controller **Pi venv:** Make sure to activate: `source .venv/bin/activate` - +
--- -## Submission Checklist +
+

Submission Checklist

Before submitting: - [ ] Delete prep/instructions above @@ -238,6 +549,8 @@ Before submitting: **Your README = story of what YOU built!** +
+ --- Resources: [MQTT Guide](https://www.hivemq.com/mqtt-essentials/) | [Paho Python](https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php) | [Flask-SocketIO](https://flask-socketio.readthedocs.io/) diff --git a/Lab 6/WechatIMG1529.jpeg b/Lab 6/WechatIMG1529.jpeg new file mode 100644 index 0000000000..4b90337951 Binary files /dev/null and b/Lab 6/WechatIMG1529.jpeg differ diff --git a/Lab 6/diagram.jpeg b/Lab 6/diagram.jpeg new file mode 100644 index 0000000000..5c761b6c81 Binary files /dev/null and b/Lab 6/diagram.jpeg differ diff --git a/Lab 6/game_master.py b/Lab 6/game_master.py new file mode 100644 index 0000000000..a3a2824198 --- /dev/null +++ b/Lab 6/game_master.py @@ -0,0 +1,135 @@ +import random +import time +import paho.mqtt.client as mqtt + +BROKER = "farlab.infosci.cornell.edu" +PORT = 1883 +USERNAME = "idd" +PASSWORD = "device@theFarm" + +# Shared narration topic +STORY_TOPIC = "game/story" + +# define player and MQTT topic +players = [ + {"name": "joy", "task_topic": "game/joy/task", "result_topic": "game/joy/result"}, + {"name": "hester", "task_topic": "game/hester/task", "result_topic": "game/hester/result"}, + {"name": "sandy", "task_topic": "game/sandy/task", "result_topic": "game/sandy/result"} +] + +# task instance +task_bank = { + "joy": ["touch_1", "touch_3", "touch_5"], + "hester": ["joystick_up", "joystick_down", "joystick_left", "joystick_right", "joystick_press"], + "sandy": ["color_red"] +} + +waiting_for = None +game_over = False + +def broadcast_story(text): + print("\n" + text + "\n") + client.publish(STORY_TOPIC, text) + +def on_message(client, userdata, msg): + global waiting_for, game_over + + if waiting_for is None: + return + + payload = msg.payload.decode() + print(f"[RECV] {msg.topic} -> {payload}") + + if payload == "success": + waiting_for = None + else: + broadcast_story("Your action falters. The mechanism rejects your attempt. The temple seals itself. The heist has failed.") + client.publish("game/status", "game_fail") + game_over = True + +def play_game(): + global waiting_for, game_over + + game_over = False + + # Opening story + broadcast_story( + "You are part of a legendary trio of master thieves known as the Silent Serpents. " + "You have infiltrated the Temple of the Sleeping Star in search of the Heart of Dawn. " + "The temple is protected by layered traps that require perfect cooperation to overcome." + ) + + assigned_tasks = { + p["name"]: random.choice(task_bank[p["name"]]) + for p in players + } + + print("\nGenerated Tasks:") + for p in players: + print(f" {p['name']} must perform {assigned_tasks[p['name']]}") + + # Player order story + execution + for p in players: + if game_over: + return + + task = assigned_tasks[p["name"]] + + if p["name"] == "joy": + broadcast_story( + "A wall of ancient runes glows faintly. The correct touch will unlock the mechanism. " + "Choose wisely, for a wrong move will seal the chamber forever." + ) + + elif p["name"] == "hester": + broadcast_story( + "A shifting stone walkway activates beneath your feet. You must step in the correct direction " + "to avoid the concealed spikes below." + ) + + elif p["name"] == "sandy": + broadcast_story( + "A spectral barrier blocks your passage. Its surface ripples with shifting color. " + "Only by matching the correct hue can the barrier dissolve." + ) + + print(f"Sending task to {p['name']} : {task}") + client.publish(p["task_topic"], task) + + waiting_for = p["name"] + + # wait for success + t = time.time() + while waiting_for is not None and time.time() - t < 15: + time.sleep(0.1) + + if waiting_for is not None: + broadcast_story("Time slips away. The mechanism locks permanently. The heist ends here.") + client.publish("game/status", "game_fail") + return + + broadcast_story("Your action is precise. The mechanism responds. The path forward opens.") + + broadcast_story( + "The final seal breaks. The chamber reveals the Heart of Dawn, shimmering in the darkness. " + "You lift the relic and escape into the night. The Silent Serpents succeed once again." + ) + client.publish("game/status", "game_success") + + +# MQTT Setup +client = mqtt.Client() +client.username_pw_set(USERNAME, PASSWORD) +client.on_message = on_message +client.connect(BROKER, PORT, 60) + +for p in players: + client.subscribe(p["result_topic"]) + +client.loop_start() + +print("===== Game Master Started =====") + +while True: + input("\nPress Enter to begin the next attempt...") + play_game() diff --git a/Lab 6/pixels.png b/Lab 6/pixels.png new file mode 100644 index 0000000000..8df6491dc9 Binary files /dev/null and b/Lab 6/pixels.png differ diff --git a/Lab 6/player_A.py b/Lab 6/player_A.py new file mode 100644 index 0000000000..c92570f4df --- /dev/null +++ b/Lab 6/player_A.py @@ -0,0 +1,58 @@ +import time +import board +import busio +import paho.mqtt.client as mqtt +import adafruit_mpr121 + +BROKER = "farlab.infosci.cornell.edu" +PORT = 1883 +USERNAME = "idd" +PASSWORD = "device@theFarm" + +TASK_TOPIC = "game/joy/task" +RESULT_TOPIC = "game/joy/result" + + +i2c = busio.I2C(board.SCL, board.SDA) +touch_sensor = adafruit_mpr121.MPR121(i2c) + +current_task = None +task_start_time = 0 + +def on_message(client, userdata, msg): + global current_task, task_start_time + payload = msg.payload.decode() + print(f"[TASK RECEIVED] {payload}") + + # payload exp: "touch_3" + if payload.startswith("touch_"): + current_task = int(payload.split("_")[1]) + task_start_time = time.time() + +client = mqtt.Client() +client.username_pw_set(USERNAME, PASSWORD) +client.on_message = on_message +client.connect(BROKER, PORT, 60) + +client.subscribe(TASK_TOPIC) +client.loop_start() + +print("=== Player A (joy) READY ===") +print("waiting for the task...") + +while True: + if current_task is not None: + # time out + if time.time() - task_start_time > 8: + print(" Timeout, FAIL") + client.publish(RESULT_TOPIC, "fail") + current_task = None + continue + + # monitor touch + if touch_sensor[current_task].value: + print(f" Touch {current_task} detected, SUCCESS") + client.publish(RESULT_TOPIC, "success") + current_task = None + + time.sleep(0.05) diff --git a/Lab 6/setup1.jpeg b/Lab 6/setup1.jpeg new file mode 100644 index 0000000000..fa7b09378c Binary files /dev/null and b/Lab 6/setup1.jpeg differ diff --git a/Lab 6/setup2.jpeg b/Lab 6/setup2.jpeg new file mode 100644 index 0000000000..4b90337951 Binary files /dev/null and b/Lab 6/setup2.jpeg differ diff --git a/Lab 6/storyboard.png b/Lab 6/storyboard.png new file mode 100644 index 0000000000..17dd0e71c9 Binary files /dev/null and b/Lab 6/storyboard.png differ diff --git a/README.md b/README.md index 086eafada8..77a8745c3b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# [Your name here]'s-Lab-Hub +# [Joy Sun]'s-Lab-Hub for [Interactive Device Design](https://github.com/FAR-Lab/Developing-and-Designing-Interactive-Devices/) Please place links here to the README.md's for each of your labs here: