1. 生态系统概述
1.1 Hadoop生态系统简介
Hadoop生态系统是一个庞大的开源软件集合，围绕Apache Hadoop核心组件构建，为大数据存储、处理、分析和管理提供了完整的解决方案。
from typing import Dict, List, Any, Optional
from dataclasses import dataclass
from enum import Enum
from datetime import datetime
class ComponentCategory(Enum):
"""组件类别"""
STORAGE = "存储"
PROCESSING = "处理"
COORDINATION = "协调"
WORKFLOW = "工作流"
MONITORING = "监控"
SECURITY = "安全"
DATA_INGESTION = "数据摄取"
QUERY_ENGINE = "查询引擎"
MACHINE_LEARNING = "机器学习"
STREAMING = "流处理"
class MaturityLevel(Enum):
"""成熟度级别"""
INCUBATING = "孵化中"
STABLE = "稳定"
MATURE = "成熟"
DEPRECATED = "已弃用"
@dataclass
class EcosystemComponent:
"""生态系统组件"""
name: str
category: ComponentCategory
description: str
key_features: List[str]
use_cases: List[str]
maturity: MaturityLevel
dependencies: List[str]
alternatives: List[str]
learning_curve: str # 学习曲线：简单、中等、困难
popularity_score: int # 流行度评分 1-10
class HadoopEcosystemGuide:
"""
Hadoop生态系统指南
"""
def __init__(self):
self.components = self._initialize_components()
self.integration_patterns = self._initialize_integration_patterns()
self.architecture_templates = self._initialize_architecture_templates()
def _initialize_components(self) -> List[EcosystemComponent]:
"""
初始化生态系统组件
Returns:
List[EcosystemComponent]: 组件列表
"""
components = [
# 存储组件
EcosystemComponent(
name="HDFS",
category=ComponentCategory.STORAGE,
description="Hadoop分布式文件系统，提供高容错性的分布式存储",
key_features=[
"分布式存储",
"高容错性",
"高吞吐量",
"数据复制",
"大文件优化"
],
use_cases=[
"大数据存储",
"数据湖构建",
"批处理数据源",
"数据归档"
],
maturity=MaturityLevel.MATURE,
dependencies=["Java"],
alternatives=["Amazon S3", "Google Cloud Storage", "Azure Blob Storage"],
learning_curve="中等",
popularity_score=9
),
EcosystemComponent(
name="HBase",
category=ComponentCategory.STORAGE,
description="基于HDFS的分布式、面向列的NoSQL数据库",
key_features=[
"列式存储",
"实时读写",
"自动分片",
"强一致性",
"与Hadoop集成"
],
use_cases=[
"实时数据访问",
"时序数据存储",
"大规模结构化数据",
"在线服务后端"
],
maturity=MaturityLevel.MATURE,
dependencies=["HDFS", "ZooKeeper"],
alternatives=["Cassandra", "MongoDB", "DynamoDB"],
learning_curve="困难",
popularity_score=7
),
# 处理组件
EcosystemComponent(
name="Apache Spark",
category=ComponentCategory.PROCESSING,
description="统一的大数据处理引擎，支持批处理、流处理、机器学习和图计算",
key_features=[
"内存计算",
"多语言支持",
"统一API",
"流批一体",
"机器学习库"
],
use_cases=[
"大数据分析",
"实时流处理",
"机器学习",
"ETL处理",
"图计算"
],
maturity=MaturityLevel.MATURE,
dependencies=["Scala", "Java", "Python"],
alternatives=["Apache Flink", "Apache Storm", "MapReduce"],
learning_curve="中等",
popularity_score=10
),
EcosystemComponent(
name="Apache Flink",
category=ComponentCategory.STREAMING,
description="分布式流处理引擎，提供低延迟、高吞吐量的流处理能力",
key_features=[
"真正的流处理",
"事件时间处理",
"状态管理",
"容错机制",
"精确一次语义"
],
use_cases=[
"实时数据处理",
"事件驱动应用",
"实时分析",
"复杂事件处理"
],
maturity=MaturityLevel.STABLE,
dependencies=["Java", "Scala"],
alternatives=["Apache Spark Streaming", "Apache Storm", "Apache Kafka Streams"],
learning_curve="困难",
popularity_score=8
),
# 查询引擎
EcosystemComponent(
name="Apache Hive",
category=ComponentCategory.QUERY_ENGINE,
description="基于Hadoop的数据仓库软件，提供SQL查询接口",
key_features=[
"SQL接口",
"元数据管理",
"多种存储格式",
"分区支持",
"UDF支持"
],
use_cases=[
"数据仓库",
"批量数据分析",
"ETL处理",
"报表生成"
],
maturity=MaturityLevel.MATURE,
dependencies=["HDFS", "MapReduce/Tez/Spark"],
alternatives=["Apache Impala", "Presto", "Apache Drill"],
learning_curve="简单",
popularity_score=8
),
EcosystemComponent(
name="Apache Impala",
category=ComponentCategory.QUERY_ENGINE,
description="高性能的分布式SQL查询引擎，专为Hadoop设计",
key_features=[
"MPP架构",
"内存计算",
"实时查询",
"标准SQL",
"与Hadoop集成"
],
use_cases=[
"交互式查询",
"实时分析",
"商业智能",
"即席查询"
],
maturity=MaturityLevel.STABLE,
dependencies=["HDFS", "HBase", "Hive Metastore"],
alternatives=["Presto", "Apache Drill", "Spark SQL"],
learning_curve="中等",
popularity_score=7
),
# 协调组件
EcosystemComponent(
name="Apache ZooKeeper",
category=ComponentCategory.COORDINATION,
description="分布式协调服务，为分布式应用提供配置管理、命名服务等",
key_features=[
"配置管理",
"命名服务",
"分布式锁",
"集群管理",
"通知机制"
],
use_cases=[
"配置中心",
"服务发现",
"分布式锁",
"集群协调",
"主从选举"
],
maturity=MaturityLevel.MATURE,
dependencies=["Java"],
alternatives=["Apache Curator", "etcd", "Consul"],
learning_curve="中等",
popularity_score=8
),
# 数据摄取
EcosystemComponent(
name="Apache Kafka",
category=ComponentCategory.DATA_INGESTION,
description="分布式流处理平台，提供高吞吐量的消息队列服务",
key_features=[
"高吞吐量",
"持久化存储",
"分布式架构",
"实时处理",
"多消费者支持"
],
use_cases=[
"消息队列",
"日志收集",
"事件流",
"数据管道",
"实时数据传输"
],
maturity=MaturityLevel.MATURE,
dependencies=["Java", "ZooKeeper"],
alternatives=["Apache Pulsar", "RabbitMQ", "Amazon Kinesis"],
learning_curve="中等",
popularity_score=9
),
EcosystemComponent(
name="Apache Flume",
category=ComponentCategory.DATA_INGESTION,
description="分布式日志收集系统，专为Hadoop环境设计",
key_features=[
"可靠数据传输",
"灵活配置",
"多种数据源",
"事务支持",
"负载均衡"
],
use_cases=[
"日志收集",
"数据摄取",
"实时数据传输",
"ETL前端"
],
maturity=MaturityLevel.STABLE,
dependencies=["Java"],
alternatives=["Apache Kafka", "Logstash", "Filebeat"],
learning_curve="简单",
popularity_score=6
),
# 工作流
EcosystemComponent(
name="Apache Oozie",
category=ComponentCategory.WORKFLOW,
description="Hadoop作业调度系统，管理复杂的数据处理工作流",
key_features=[
"工作流调度",
"依赖管理",
"错误处理",
"Web界面",
"多种作业类型"
],
use_cases=[
"ETL流程",
"批处理调度",
"数据管道",
"作业编排"
],
maturity=MaturityLevel.STABLE,
dependencies=["Hadoop"],
alternatives=["Apache Airflow", "Azkaban", "Luigi"],
learning_curve="中等",
popularity_score=5
),
# 监控
EcosystemComponent(
name="Apache Ambari",
category=ComponentCategory.MONITORING,
description="Hadoop集群管理和监控平台",
key_features=[
"集群管理",
"服务监控",
"配置管理",
"告警系统",
"Web界面"
],
use_cases=[
"集群部署",
"运维监控",
"配置管理",
"性能监控"
],
maturity=MaturityLevel.STABLE,
dependencies=["Python", "Java"],
alternatives=["Cloudera Manager", "Hortonworks Data Platform"],
learning_curve="中等",
popularity_score=6
),
# 机器学习
EcosystemComponent(
name="Apache Mahout",
category=ComponentCategory.MACHINE_LEARNING,
description="可扩展的机器学习库，专为大数据环境设计",
key_features=[
"分布式算法",
"可扩展性",
"多种算法",
"与Hadoop集成",
"数学库"
],
use_cases=[
"推荐系统",
"聚类分析",
"分类算法",
"协同过滤"
],
maturity=MaturityLevel.STABLE,
dependencies=["Hadoop", "Spark"],
alternatives=["Spark MLlib", "TensorFlow", "Scikit-learn"],
learning_curve="困难",
popularity_score=4
)
]
return components
def _initialize_integration_patterns(self) -> Dict[str, Dict[str, Any]]:
"""
初始化集成模式
Returns:
Dict[str, Dict[str, Any]]: 集成模式
"""
patterns = {
"Lambda架构": {
"description": "结合批处理和流处理的混合架构",
"components": ["Kafka", "Spark", "HBase", "HDFS"],
"use_cases": ["实时分析", "历史数据处理", "低延迟查询"],
"advantages": ["容错性强", "支持复杂查询", "数据一致性"],
"disadvantages": ["复杂度高", "维护成本大", "数据重复"]
},
"Kappa架构": {
"description": "纯流处理架构，统一批处理和流处理",
"components": ["Kafka", "Flink", "Elasticsearch"],
"use_cases": ["实时处理", "事件驱动", "简化架构"],
"advantages": ["架构简单", "实时性好", "维护容易"],
"disadvantages": ["历史数据处理复杂", "状态管理挑战"]
},
"数据湖架构": {
"description": "集中存储结构化和非结构化数据的架构",
"components": ["HDFS", "Spark", "Hive", "HBase"],
"use_cases": ["数据存储", "数据探索", "机器学习"],
"advantages": ["灵活性高", "成本低", "支持多种数据类型"],
"disadvantages": ["数据质量挑战", "治理复杂"]
},
"现代数据栈": {
"description": "云原生的现代数据处理架构",
"components": ["Kafka", "Spark", "Delta Lake", "Kubernetes"],
"use_cases": ["云端数据处理", "DevOps集成", "弹性扩展"],
"advantages": ["云原生", "弹性扩展", "运维简化"],
"disadvantages": ["云厂商绑定", "成本控制"]
}
}
return patterns
def _initialize_architecture_templates(self) -> Dict[str, Dict[str, Any]]:
"""
初始化架构模板
Returns:
Dict[str, Dict[str, Any]]: 架构模板
"""
templates = {
"基础大数据平台": {
"description": "适合初学者的基础Hadoop平台",
"core_components": ["HDFS", "YARN", "MapReduce", "Hive"],
"optional_components": ["HBase", "ZooKeeper", "Oozie"],
"complexity": "简单",
"use_cases": ["数据仓库", "批处理", "报表分析"],
"deployment_size": "小型（3-10节点）"
},
"实时分析平台": {
"description": "支持实时数据处理和分析的平台",
"core_components": ["Kafka", "Spark Streaming", "HBase", "Elasticsearch"],
"optional_components": ["Flink", "Redis", "Grafana"],
"complexity": "中等",
"use_cases": ["实时监控", "事件处理", "在线推荐"],
"deployment_size": "中型（10-50节点）"
},
"机器学习平台": {
"description": "支持大规模机器学习的数据平台",
"core_components": ["Spark", "HDFS", "Jupyter", "MLflow"],
"optional_components": ["Kubeflow", "TensorFlow", "PyTorch"],
"complexity": "困难",
"use_cases": ["模型训练", "特征工程", "模型部署"],
"deployment_size": "大型（50+节点）"
},
"数据湖平台": {
"description": "统一存储和处理多种数据类型的平台",
"core_components": ["HDFS", "Spark", "Hive", "Presto"],
"optional_components": ["Delta Lake", "Apache Iceberg", "Ranger"],
"complexity": "中等",
"use_cases": ["数据探索", "自助分析", "数据科学"],
"deployment_size": "中大型（20-100节点）"
}
}
return templates
def get_components_by_category(self, category: ComponentCategory) -> List[EcosystemComponent]:
"""
按类别获取组件
Args:
category: 组件类别
Returns:
List[EcosystemComponent]: 组件列表
"""
return [comp for comp in self.components if comp.category == category]
def get_component_by_name(self, name: str) -> Optional[EcosystemComponent]:
"""
按名称获取组件
Args:
name: 组件名称
Returns:
Optional[EcosystemComponent]: 组件信息
"""
return next((comp for comp in self.components if comp.name == name), None)
def recommend_components(self, use_case: str, complexity_preference: str = "中等") -> List[Dict[str, Any]]:
"""
根据用例推荐组件
Args:
use_case: 使用场景
complexity_preference: 复杂度偏好
Returns:
List[Dict[str, Any]]: 推荐组件列表
"""
recommendations = []
for component in self.components:
# 检查用例匹配
use_case_match = any(use_case.lower() in uc.lower() for uc in component.use_cases)
# 检查复杂度匹配
complexity_match = component.learning_curve == complexity_preference
if use_case_match:
score = component.popularity_score
if complexity_match:
score += 2
recommendations.append({
'component': component,
'score': score,
'reason': f"适用于{use_case}，学习曲线{component.learning_curve}"
})
# 按评分排序
recommendations.sort(key=lambda x: x['score'], reverse=True)
return recommendations[:5] # 返回前5个推荐
def generate_architecture_recommendation(self, requirements: Dict[str, Any]) -> Dict[str, Any]:
"""
生成架构推荐
Args:
requirements: 需求描述
Returns:
Dict[str, Any]: 架构推荐
"""
use_cases = requirements.get('use_cases', [])
data_volume = requirements.get('data_volume', 'medium') # small, medium, large
real_time_requirement = requirements.get('real_time', False)
complexity_tolerance = requirements.get('complexity', '中等')
recommended_components = []
# 基础组件（总是需要）
recommended_components.extend(['HDFS', 'YARN'])
# 根据用例添加组件
if 'batch_processing' in use_cases or 'data_warehouse' in use_cases:
recommended_components.extend(['Hive', 'Spark'])
if real_time_requirement:
recommended_components.extend(['Kafka', 'Spark Streaming'])
if complexity_tolerance == '困难':
recommended_components.append('Flink')
if 'machine_learning' in use_cases:
recommended_components.extend(['Spark', 'Jupyter'])
if 'nosql' in use_cases or 'real_time_access' in use_cases:
recommended_components.append('HBase')
if len(recommended_components) > 5: # 复杂系统需要协调
recommended_components.append('ZooKeeper')
# 选择架构模板
template = None
if real_time_requirement:
template = self.architecture_templates['实时分析平台']
elif 'machine_learning' in use_cases:
template = self.architecture_templates['机器学习平台']
elif data_volume == 'large':
template = self.architecture_templates['数据湖平台']
else:
template = self.architecture_templates['基础大数据平台']
return {
'recommended_components': list(set(recommended_components)),
'architecture_template': template,
'integration_pattern': self._suggest_integration_pattern(requirements),
'deployment_considerations': self._generate_deployment_considerations(requirements),
'learning_path': self._generate_learning_path(recommended_components)
}
def _suggest_integration_pattern(self, requirements: Dict[str, Any]) -> Dict[str, Any]:
"""
建议集成模式
Args:
requirements: 需求描述
Returns:
Dict[str, Any]: 集成模式建议
"""
if requirements.get('real_time', False) and requirements.get('batch_processing', False):
return self.integration_patterns['Lambda架构']
elif requirements.get('real_time', False):
return self.integration_patterns['Kappa架构']
elif requirements.get('data_exploration', False):
return self.integration_patterns['数据湖架构']
else:
return self.integration_patterns['现代数据栈']
def _generate_deployment_considerations(self, requirements: Dict[str, Any]) -> List[str]:
"""
生成部署考虑事项
Args:
requirements: 需求描述
Returns:
List[str]: 部署考虑事项
"""
considerations = []
data_volume = requirements.get('data_volume', 'medium')
if data_volume == 'large':
considerations.append("考虑使用SSD存储提高I/O性能")
considerations.append("规划足够的网络带宽")
considerations.append("考虑数据分层存储策略")
if requirements.get('real_time', False):
considerations.append("优化网络延迟")
considerations.append("配置足够的内存")
considerations.append("考虑使用SSD作为缓存")
if requirements.get('high_availability', False):
considerations.append("部署多个数据中心")
considerations.append("配置组件高可用")
considerations.append("建立灾备方案")
considerations.extend([
"制定监控和告警策略",
"规划容量增长",
"建立安全策略",
"制定备份和恢复计划"
])
return considerations
def _generate_learning_path(self, components: List[str]) -> List[Dict[str, str]]:
"""
生成学习路径
Args:
components: 组件列表
Returns:
List[Dict[str, str]]: 学习路径
"""
learning_order = {
'HDFS': 1,
'YARN': 2,
'MapReduce': 3,
'Hive': 4,
'Spark': 5,
'HBase': 6,
'Kafka': 7,
'ZooKeeper': 8,
'Flink': 9
}
# 按学习顺序排序
sorted_components = sorted(
[comp for comp in components if comp in learning_order],
key=lambda x: learning_order.get(x, 999)
)
learning_path = []
for i, comp in enumerate(sorted_components, 1):
component_obj = self.get_component_by_name(comp)
if component_obj:
learning_path.append({
'step': i,
'component': comp,
'difficulty': component_obj.learning_curve,
'estimated_time': self._estimate_learning_time(component_obj.learning_curve),
'prerequisites': component_obj.dependencies
})
return learning_path
def _estimate_learning_time(self, difficulty: str) -> str:
"""
估算学习时间
Args:
difficulty: 难度级别
Returns:
str: 估算学习时间
"""
time_mapping = {
'简单': '1-2周',
'中等': '2-4周',
'困难': '4-8周'
}
return time_mapping.get(difficulty, '2-4周')
def generate_ecosystem_report(self) -> Dict[str, Any]:
"""
生成生态系统报告
Returns:
Dict[str, Any]: 生态系统报告
"""
category_stats = {}
for category in ComponentCategory:
components = self.get_components_by_category(category)
category_stats[category.value] = {
'count': len(components),
'components': [comp.name for comp in components],
'avg_popularity': sum(comp.popularity_score for comp in components) / len(components) if components else 0
}
maturity_stats = {}
for maturity in MaturityLevel:
components = [comp for comp in self.components if comp.maturity == maturity]
maturity_stats[maturity.value] = len(components)
return {
'total_components': len(self.components),
'category_distribution': category_stats,
'maturity_distribution': maturity_stats,
'top_components': sorted(
self.components,
key=lambda x: x.popularity_score,
reverse=True
)[:5],
'integration_patterns': list(self.integration_patterns.keys()),
'architecture_templates': list(self.architecture_templates.keys())
}
# 使用示例
if __name__ == "__main__":
# 创建生态系统指南
ecosystem = HadoopEcosystemGuide()
print("=== Hadoop生态系统组件指南 ===")
# 按类别显示组件
print("\n=== 按类别显示组件 ===")
for category in ComponentCategory:
components = ecosystem.get_components_by_category(category)
if components:
print(f"\n{category.value}:")
for comp in components:
print(f" - {comp.name}: {comp.description}")
# 组件推荐示例
print("\n=== 组件推荐示例 ===")
recommendations = ecosystem.recommend_components("实时处理", "中等")
print("针对'实时处理'用例的推荐:")
for rec in recommendations:
print(f" {rec['component'].name} (评分: {rec['score']}) - {rec['reason']}")
# 架构推荐示例
print("\n=== 架构推荐示例 ===")
requirements = {
'use_cases': ['batch_processing', 'real_time'],
'data_volume': 'large',
'real_time': True,
'complexity': '中等',
'high_availability': True
}
arch_rec = ecosystem.generate_architecture_recommendation(requirements)
print("推荐组件:", arch_rec['recommended_components'])
print("架构模板:", arch_rec['architecture_template']['description'])
print("集成模式:", arch_rec['integration_pattern']['description'])
print("\n部署考虑事项:")
for consideration in arch_rec['deployment_considerations']:
print(f" - {consideration}")
print("\n学习路径:")
for step in arch_rec['learning_path']:
print(f" {step['step']}. {step['component']} ({step['difficulty']}, 预计{step['estimated_time']})")
# 生态系统报告
print("\n=== 生态系统报告 ===")
report = ecosystem.generate_ecosystem_report()
print(f"总组件数: {report['total_components']}")
print("\n类别分布:")
for category, stats in report['category_distribution'].items():
print(f" {category}: {stats['count']}个组件 (平均流行度: {stats['avg_popularity']:.1f})")
print("\n成熟度分布:")
for maturity, count in report['maturity_distribution'].items():
print(f" {maturity}: {count}个组件")
print("\n最受欢迎的组件:")
for comp in report['top_components']:
print(f" {comp.name} (流行度: {comp.popularity_score})")
1.2 生态系统架构层次
from typing import Dict, List, Tuple
from dataclasses import dataclass
from enum import Enum
class ArchitectureLayer(Enum):
"""架构层次"""
STORAGE = "存储层"
RESOURCE_MANAGEMENT = "资源管理层"
PROCESSING = "处理层"
COORDINATION = "协调层"
APPLICATION = "应用层"
INTERFACE = "接口层"
@dataclass
class LayerComponent:
"""层次组件"""
name: str
layer: ArchitectureLayer
description: str
responsibilities: List[str]
interfaces: List[str]
dependencies: List[str]
class EcosystemArchitecture:
"""
生态系统架构
"""
def __init__(self):
self.layers = self._initialize_layers()
self.component_relationships = self._initialize_relationships()
def _initialize_layers(self) -> Dict[ArchitectureLayer, List[LayerComponent]]:
"""
初始化架构层次
Returns:
Dict[ArchitectureLayer, List[LayerComponent]]: 层次组件映射
"""
layers = {
ArchitectureLayer.STORAGE: [
LayerComponent(
name="HDFS",
layer=ArchitectureLayer.STORAGE,
description="分布式文件系统",
responsibilities=["数据存储", "数据复制", "容错处理"],
interfaces=["WebHDFS", "Java API", "命令行"],
dependencies=["Java"]
),
LayerComponent(
name="HBase",
layer=ArchitectureLayer.STORAGE,
description="NoSQL数据库",
responsibilities=["实时数据访问", "列式存储", "自动分片"],
interfaces=["Java API", "REST API", "Thrift"],
dependencies=["HDFS", "ZooKeeper"]
)
],
ArchitectureLayer.RESOURCE_MANAGEMENT: [
LayerComponent(
name="YARN",
layer=ArchitectureLayer.RESOURCE_MANAGEMENT,
description="资源管理器",
responsibilities=["资源分配", "作业调度", "集群管理"],
interfaces=["ResourceManager API", "Web UI"],
dependencies=["HDFS"]
)
],
ArchitectureLayer.PROCESSING: [
LayerComponent(
name="MapReduce",
layer=ArchitectureLayer.PROCESSING,
description="批处理框架",
responsibilities=["批量数据处理", "分布式计算"],
interfaces=["Java API", "Streaming API"],
dependencies=["YARN", "HDFS"]
),
LayerComponent(
name="Spark",
layer=ArchitectureLayer.PROCESSING,
description="统一计算引擎",
responsibilities=["批处理", "流处理", "机器学习", "图计算"],
interfaces=["Scala API", "Python API", "Java API", "R API"],
dependencies=["YARN", "HDFS"]
)
],
ArchitectureLayer.COORDINATION: [
LayerComponent(
name="ZooKeeper",
layer=ArchitectureLayer.COORDINATION,
description="分布式协调服务",
responsibilities=["配置管理", "命名服务", "分布式锁"],
interfaces=["Java API", "C API", "命令行"],
dependencies=["Java"]
)
],
ArchitectureLayer.APPLICATION: [
LayerComponent(
name="Hive",
layer=ArchitectureLayer.APPLICATION,
description="数据仓库软件",
responsibilities=["SQL查询", "元数据管理", "数据仓库"],
interfaces=["HiveQL", "JDBC", "ODBC"],
dependencies=["HDFS", "MapReduce/Spark"]
),
LayerComponent(
name="HBase",
layer=ArchitectureLayer.APPLICATION,
description="NoSQL数据库",
responsibilities=["实时数据访问", "大规模数据存储"],
interfaces=["Java API", "REST API", "Shell"],
dependencies=["HDFS", "ZooKeeper"]
)
],
ArchitectureLayer.INTERFACE: [
LayerComponent(
name="Ambari",
layer=ArchitectureLayer.INTERFACE,
description="集群管理界面",
responsibilities=["集群监控", "配置管理", "服务管理"],
interfaces=["Web UI", "REST API"],
dependencies=["所有Hadoop组件"]
)
]
}
return layers
def _initialize_relationships(self) -> Dict[str, List[str]]:
"""
初始化组件关系
Returns:
Dict[str, List[str]]: 组件依赖关系
"""
relationships = {
"HDFS": [], # 基础层，无依赖
"YARN": ["HDFS"],
"MapReduce": ["YARN", "HDFS"],
"Spark": ["YARN", "HDFS"],
"Hive": ["HDFS", "MapReduce"],
"HBase": ["HDFS", "ZooKeeper"],
"ZooKeeper": [],
"Kafka": ["ZooKeeper"],
"Flume": ["HDFS"],
"Oozie": ["YARN", "HDFS"],
"Ambari": ["所有组件"]
}
return relationships
def get_layer_components(self, layer: ArchitectureLayer) -> List[LayerComponent]:
"""
获取指定层的组件
Args:
layer: 架构层次
Returns:
List[LayerComponent]: 组件列表
"""
return self.layers.get(layer, [])
def get_component_dependencies(self, component_name: str) -> List[str]:
"""
获取组件依赖
Args:
component_name: 组件名称
Returns:
List[str]: 依赖组件列表
"""
return self.component_relationships.get(component_name, [])
def generate_deployment_order(self, components: List[str]) -> List[List[str]]:
"""
生成部署顺序
Args:
components: 要部署的组件列表
Returns:
List[List[str]]: 按层次分组的部署顺序
"""
# 拓扑排序算法
in_degree = {comp: 0 for comp in components}
graph = {comp: [] for comp in components}
# 构建图和计算入度
for comp in components:
deps = self.get_component_dependencies(comp)
for dep in deps:
if dep in components:
graph[dep].append(comp)
in_degree[comp] += 1
# 拓扑排序
result = []
queue = [comp for comp in components if in_degree[comp] == 0]
while queue:
current_level = queue[:]
queue = []
result.append(current_level)
for comp in current_level:
for neighbor in graph[comp]:
in_degree[neighbor] -= 1
if in_degree[neighbor] == 0:
queue.append(neighbor)
return result
def analyze_architecture_complexity(self, components: List[str]) -> Dict[str, Any]:
"""
分析架构复杂度
Args:
components: 组件列表
Returns:
Dict[str, Any]: 复杂度分析结果
"""
# 计算层次分布
layer_distribution = {}
for layer in ArchitectureLayer:
layer_components = [comp.name for comp in self.get_layer_components(layer)]
count = len([comp for comp in components if comp in layer_components])
if count > 0:
layer_distribution[layer.value] = count
# 计算依赖复杂度
total_dependencies = sum(
len(self.get_component_dependencies(comp))
for comp in components
)
# 计算复杂度评分
complexity_score = len(components) + total_dependencies
if complexity_score <= 5:
complexity_level = "简单"
elif complexity_score <= 15:
complexity_level = "中等"
else:
complexity_level = "复杂"
return {
'total_components': len(components),
'layer_distribution': layer_distribution,
'total_dependencies': total_dependencies,
'complexity_score': complexity_score,
'complexity_level': complexity_level,
'deployment_phases': len(self.generate_deployment_order(components))
}
def generate_architecture_diagram_data(self, components: List[str]) -> Dict[str, Any]:
"""
生成架构图数据
Args:
components: 组件列表
Returns:
Dict[str, Any]: 架构图数据
"""
nodes = []
edges = []
# 生成节点
for comp in components:
# 查找组件所属层次
layer = None
for arch_layer, layer_components in self.layers.items():
if any(lc.name == comp for lc in layer_components):
layer = arch_layer.value
break
nodes.append({
'id': comp,
'label': comp,
'layer': layer or "未知",
'dependencies': self.get_component_dependencies(comp)
})
# 生成边
for comp in components:
deps = self.get_component_dependencies(comp)
for dep in deps:
if dep in components:
edges.append({
'from': dep,
'to': comp,
'type': 'dependency'
})
return {
'nodes': nodes,
'edges': edges,
'layers': list(set(node['layer'] for node in nodes))
}
# 使用示例
if __name__ == "__main__":
# 创建架构分析器
architecture = EcosystemArchitecture()
print("=== Hadoop生态系统架构分析 ===")
# 显示各层组件
print("\n=== 架构层次 ===")
for layer in ArchitectureLayer:
components = architecture.get_layer_components(layer)
if components:
print(f"\n{layer.value}:")
for comp in components:
print(f" - {comp.name}: {comp.description}")
print(f" 职责: {', '.join(comp.responsibilities)}")
print(f" 接口: {', '.join(comp.interfaces)}")
# 分析示例架构
example_components = ["HDFS", "YARN", "Spark", "Hive", "HBase", "ZooKeeper"]
print("\n=== 部署顺序分析 ===")
deployment_order = architecture.generate_deployment_order(example_components)
for i, phase in enumerate(deployment_order, 1):
print(f"阶段 {i}: {', '.join(phase)}")
print("\n=== 架构复杂度分析 ===")
complexity = architecture.analyze_architecture_complexity(example_components)
print(f"总组件数: {complexity['total_components']}")
print(f"总依赖数: {complexity['total_dependencies']}")
print(f"复杂度评分: {complexity['complexity_score']}")
print(f"复杂度级别: {complexity['complexity_level']}")
print(f"部署阶段数: {complexity['deployment_phases']}")
print("\n层次分布:")
for layer, count in complexity['layer_distribution'].items():
print(f" {layer}: {count}个组件")
print("\n=== 架构图数据 ===")
diagram_data = architecture.generate_architecture_diagram_data(example_components)
print(f"节点数: {len(diagram_data['nodes'])}")
print(f"边数: {len(diagram_data['edges'])}")
print(f"涉及层次: {', '.join(diagram_data['layers'])}")
2. 核心存储组件
2.1 HDFS深入解析
我们在前面的章节已经详细介绍了HDFS，这里重点关注其在生态系统中的作用和与其他组件的集成。
2.2 HBase详解
from typing import Dict, List, Any, Optional, Tuple
from dataclasses import dataclass
from datetime import datetime
import json
@dataclass
class HBaseTable:
"""HBase表结构"""
name: str
column_families: List[str]
regions: int
compression: str
bloom_filter: bool
max_versions: int
@dataclass
class HBaseRegion:
"""HBase区域"""
table_name: str
start_key: str
end_key: str
region_server: str
size_mb: float
read_requests: int
write_requests: int
class HBaseClusterManager:
"""
HBase集群管理器
"""
def __init__(self):
self.tables = {}
self.regions = {}
self.region_servers = {}
self.performance_metrics = {}
def create_table(self, table_config: Dict[str, Any]) -> Dict[str, Any]:
"""
创建HBase表
Args:
table_config: 表配置
Returns:
Dict[str, Any]: 创建结果
"""
table = HBaseTable(
name=table_config['name'],
column_families=table_config.get('column_families', ['cf1']),
regions=table_config.get('regions', 1),
compression=table_config.get('compression', 'SNAPPY'),
bloom_filter=table_config.get('bloom_filter', True),
max_versions=table_config.get('max_versions', 3)
)
self.tables[table.name] = table
# 创建初始区域
self._create_initial_regions(table)
return {
'status': 'success',
'table_name': table.name,
'column_families': table.column_families,
'initial_regions': table.regions,
'configuration': {
'compression': table.compression,
'bloom_filter': table.bloom_filter,
'max_versions': table.max_versions
}
}
def _create_initial_regions(self, table: HBaseTable):
"""
创建初始区域
Args:
table: HBase表
"""
if table.name not in self.regions:
self.regions[table.name] = []
# 创建预分区
for i in range(table.regions):
start_key = f"{i:08d}" if i > 0 else ""
end_key = f"{i+1:08d}" if i < table.regions - 1 else ""
region = HBaseRegion(
table_name=table.name,
start_key=start_key,
end_key=end_key,
region_server=f"rs{i % 3 + 1}", # 简单的负载均衡
size_mb=0.0,
read_requests=0,
write_requests=0
)
self.regions[table.name].append(region)
def put_data(self, table_name: str, row_key: str, column_family: str,
column: str, value: str) -> Dict[str, Any]:
"""
插入数据
Args:
table_name: 表名
row_key: 行键
column_family: 列族
column: 列名
value: 值
Returns:
Dict[str, Any]: 插入结果
"""
if table_name not in self.tables:
return {'status': 'error', 'message': f'Table {table_name} not found'}
table = self.tables[table_name]
if column_family not in table.column_families:
return {'status': 'error', 'message': f'Column family {column_family} not found'}
# 找到对应的区域
region = self._find_region(table_name, row_key)
if region:
region.write_requests += 1
region.size_mb += len(value) / (1024 * 1024) # 简化的大小计算
return {
'status': 'success',
'table': table_name,
'row_key': row_key,
'column': f'{column_family}:{column}',
'region_server': region.region_server,
'timestamp': datetime.now().isoformat()
}
return {'status': 'error', 'message': 'No suitable region found'}
def get_data(self, table_name: str, row_key: str,
column_family: str = None, column: str = None) -> Dict[str, Any]:
"""
获取数据
Args:
table_name: 表名
row_key: 行键
column_family: 列族（可选）
column: 列名（可选）
Returns:
Dict[str, Any]: 查询结果
"""
if table_name not in self.tables:
return {'status': 'error', 'message': f'Table {table_name} not found'}
# 找到对应的区域
region = self._find_region(table_name, row_key)
if region:
region.read_requests += 1
# 模拟数据返回
result = {
'status': 'success',
'table': table_name,
'row_key': row_key,
'region_server': region.region_server,
'data': self._simulate_data_retrieval(table_name, row_key, column_family, column),
'timestamp': datetime.now().isoformat()
}
return result
return {'status': 'error', 'message': 'No suitable region found'}
def _find_region(self, table_name: str, row_key: str) -> Optional[HBaseRegion]:
"""
查找行键对应的区域
Args:
table_name: 表名
row_key: 行键
Returns:
Optional[HBaseRegion]: 区域信息
"""
if table_name not in self.regions:
return None
for region in self.regions[table_name]:
# 简化的区域查找逻辑
if (not region.start_key or row_key >= region.start_key) and \
(not region.end_key or row_key < region.end_key):
return region
# 如果没找到，返回第一个区域
return self.regions[table_name][0] if self.regions[table_name] else None
def _simulate_data_retrieval(self, table_name: str, row_key: str,
column_family: str, column: str) -> Dict[str, Any]:
"""
模拟数据检索
Args:
table_name: 表名
row_key: 行键
column_family: 列族
column: 列名
Returns:
Dict[str, Any]: 模拟数据
"""
table = self.tables[table_name]
data = {}
# 如果指定了列族和列
if column_family and column:
if column_family in table.column_families:
data[f'{column_family}:{column}'] = f'value_for_{row_key}_{column}'
# 如果只指定了列族
elif column_family:
if column_family in table.column_families:
for i in range(3): # 模拟3个列
data[f'{column_family}:col{i}'] = f'value_for_{row_key}_col{i}'
# 返回所有列族的数据
else:
for cf in table.column_families:
for i in range(3):
data[f'{cf}:col{i}'] = f'value_for_{row_key}_{cf}_col{i}'
return data
def scan_table(self, table_name: str, start_row: str = None,
end_row: str = None, limit: int = 100) -> Dict[str, Any]:
"""
扫描表
Args:
table_name: 表名
start_row: 起始行键
end_row: 结束行键
limit: 限制行数
Returns:
Dict[str, Any]: 扫描结果
"""
if table_name not in self.tables:
return {'status': 'error', 'message': f'Table {table_name} not found'}
# 模拟扫描结果
results = []
for i in range(min(limit, 10)): # 模拟最多10行数据
row_key = f"row_{i:06d}"
if start_row and row_key < start_row:
continue
if end_row and row_key >= end_row:
break
row_data = self._simulate_data_retrieval(table_name, row_key, None, None)
results.append({
'row_key': row_key,
'data': row_data
})
return {
'status': 'success',
'table': table_name,
'results': results,
'count': len(results),
'scan_range': {
'start_row': start_row,
'end_row': end_row
}
}
def split_region(self, table_name: str, region_index: int, split_key: str) -> Dict[str, Any]:
"""
分裂区域
Args:
table_name: 表名
region_index: 区域索引
split_key: 分裂键
Returns:
Dict[str, Any]: 分裂结果
"""
if table_name not in self.regions:
return {'status': 'error', 'message': f'Table {table_name} not found'}
regions = self.regions[table_name]
if region_index >= len(regions):
return {'status': 'error', 'message': 'Invalid region index'}
original_region = regions[region_index]
# 创建两个新区域
region1 = HBaseRegion(
table_name=table_name,
start_key=original_region.start_key,
end_key=split_key,
region_server=original_region.region_server,
size_mb=original_region.size_mb / 2,
read_requests=0,
write_requests=0
)
region2 = HBaseRegion(
table_name=table_name,
start_key=split_key,
end_key=original_region.end_key,
region_server=self._select_region_server(),
size_mb=original_region.size_mb / 2,
read_requests=0,
write_requests=0
)
# 替换原区域
regions[region_index] = region1
regions.append(region2)
return {
'status': 'success',
'original_region': {
'start_key': original_region.start_key,
'end_key': original_region.end_key,
'size_mb': original_region.size_mb
},
'new_regions': [
{
'start_key': region1.start_key,
'end_key': region1.end_key,
'region_server': region1.region_server
},
{
'start_key': region2.start_key,
'end_key': region2.end_key,
'region_server': region2.region_server
}
],
'split_key': split_key
}
def _select_region_server(self) -> str:
"""
选择区域服务器
Returns:
str: 区域服务器名称
"""
# 简单的负载均衡策略
servers = ['rs1', 'rs2', 'rs3']
server_loads = {server: 0 for server in servers}
# 计算每个服务器的负载
for table_regions in self.regions.values():
for region in table_regions:
if region.region_server in server_loads:
server_loads[region.region_server] += 1
# 返回负载最小的服务器
return min(server_loads.items(), key=lambda x: x[1])[0]
def get_table_info(self, table_name: str) -> Dict[str, Any]:
"""
获取表信息
Args:
table_name: 表名
Returns:
Dict[str, Any]: 表信息
"""
if table_name not in self.tables:
return {'status': 'error', 'message': f'Table {table_name} not found'}
table = self.tables[table_name]
regions = self.regions.get(table_name, [])
# 计算统计信息
total_size = sum(region.size_mb for region in regions)
total_read_requests = sum(region.read_requests for region in regions)
total_write_requests = sum(region.write_requests for region in regions)
region_info = []
for i, region in enumerate(regions):
region_info.append({
'index': i,
'start_key': region.start_key,
'end_key': region.end_key,
'region_server': region.region_server,
'size_mb': round(region.size_mb, 2),
'read_requests': region.read_requests,
'write_requests': region.write_requests
})
return {
'status': 'success',
'table_name': table_name,
'column_families': table.column_families,
'configuration': {
'compression': table.compression,
'bloom_filter': table.bloom_filter,
'max_versions': table.max_versions
},
'statistics': {
'region_count': len(regions),
'total_size_mb': round(total_size, 2),
'total_read_requests': total_read_requests,
'total_write_requests': total_write_requests
},
'regions': region_info
}
def compact_table(self, table_name: str, compact_type: str = 'minor') -> Dict[str, Any]:
"""
压缩表
Args:
table_name: 表名
compact_type: 压缩类型（minor/major）
Returns:
Dict[str, Any]: 压缩结果
"""
if table_name not in self.tables:
return {'status': 'error', 'message': f'Table {table_name} not found'}
regions = self.regions.get(table_name, [])
# 模拟压缩过程
compacted_regions = 0
size_reduction = 0.0
for region in regions:
if compact_type == 'minor':
# 小压缩：减少10-20%的大小
reduction = region.size_mb * 0.15
else:
# 大压缩：减少20-40%的大小
reduction = region.size_mb * 0.30
region.size_mb = max(0, region.size_mb - reduction)
size_reduction += reduction
compacted_regions += 1
return {
'status': 'success',
'table_name': table_name,
'compact_type': compact_type,
'compacted_regions': compacted_regions,
'size_reduction_mb': round(size_reduction, 2),
'completion_time': datetime.now().isoformat()
}
def get_cluster_status(self) -> Dict[str, Any]:
"""
获取集群状态
Returns:
Dict[str, Any]: 集群状态
"""
# 统计所有表和区域
total_tables = len(self.tables)
total_regions = sum(len(regions) for regions in self.regions.values())
total_size = sum(
sum(region.size_mb for region in regions)
for regions in self.regions.values()
)
# 统计区域服务器负载
server_stats = {}
for table_regions in self.regions.values():
for region in table_regions:
server = region.region_server
if server not in server_stats:
server_stats[server] = {
'region_count': 0,
'total_size_mb': 0.0,
'read_requests': 0,
'write_requests': 0
}
server_stats[server]['region_count'] += 1
server_stats[server]['total_size_mb'] += region.size_mb
server_stats[server]['read_requests'] += region.read_requests
server_stats[server]['write_requests'] += region.write_requests
return {
'cluster_summary': {
'total_tables': total_tables,
'total_regions': total_regions,
'total_size_mb': round(total_size, 2),
'region_servers': len(server_stats)
},
'region_servers': {
server: {
'region_count': stats['region_count'],
'total_size_mb': round(stats['total_size_mb'], 2),
'read_requests': stats['read_requests'],
'write_requests': stats['write_requests']
}
for server, stats in server_stats.items()
},
'tables': list(self.tables.keys())
}
# 使用示例
if __name__ == "__main__":
# 创建HBase集群管理器
hbase = HBaseClusterManager()
print("=== HBase集群管理示例 ===")
# 创建表
print("\n=== 创建表 ===")
table_config = {
'name': 'user_profiles',
'column_families': ['personal', 'contact', 'preferences'],
'regions': 3,
'compression': 'SNAPPY',
'bloom_filter': True,
'max_versions': 5
}
result = hbase.create_table(table_config)
print(f"创建表结果: {result['status']}")
print(f"表名: {result['table_name']}")
print(f"列族: {result['column_families']}")
print(f"初始区域数: {result['initial_regions']}")
# 插入数据
print("\n=== 插入数据 ===")
put_result = hbase.put_data('user_profiles', 'user001', 'personal', 'name', 'John Doe')
print(f"插入结果: {put_result}")
put_result = hbase.put_data('user_profiles', 'user001', 'contact', 'email', 'john@example.com')
print(f"插入结果: {put_result}")
# 查询数据
print("\n=== 查询数据 ===")
get_result = hbase.get_data('user_profiles', 'user001')
print(f"查询结果: {get_result}")
# 扫描表
print("\n=== 扫描表 ===")
scan_result = hbase.scan_table('user_profiles', limit=5)
print(f"扫描结果: 找到 {scan_result['count']} 行数据")
# 获取表信息
print("\n=== 表信息 ===")
table_info = hbase.get_table_info('user_profiles')
print(f"表名: {table_info['table_name']}")
print(f"列族: {table_info['column_families']}")
print(f"区域数: {table_info['statistics']['region_count']}")
print(f"总大小: {table_info['statistics']['total_size_mb']} MB")
# 分裂区域
print("\n=== 分裂区域 ===")
split_result = hbase.split_region('user_profiles', 0, 'user500')
print(f"分裂结果: {split_result['status']}")
if split_result['status'] == 'success':
print(f"分裂键: {split_result['split_key']}")
print(f"新区域数: {len(split_result['new_regions'])}")
# 压缩表
print("\n=== 压缩表 ===")
compact_result = hbase.compact_table('user_profiles', 'major')
print(f"压缩结果: {compact_result}")
# 获取集群状态
print("\n=== 集群状态 ===")
cluster_status = hbase.get_cluster_status()
print(f"总表数: {cluster_status['cluster_summary']['total_tables']}")
print(f"总区域数: {cluster_status['cluster_summary']['total_regions']}")
print(f"区域服务器数: {cluster_status['cluster_summary']['region_servers']}")
print("\n区域服务器负载:")
for server, stats in cluster_status['region_servers'].items():
print(f" {server}: {stats['region_count']}个区域, {stats['total_size_mb']}MB")
3. 数据处理组件
3.1 Apache Spark详解
from typing import Dict, List, Any, Optional, Callable
from dataclasses import dataclass
from enum import Enum
from datetime import datetime
import json
class SparkJobStatus(Enum):
"""Spark作业状态"""
SUBMITTED = "已提交"
RUNNING = "运行中"
SUCCEEDED = "成功"
FAILED = "失败"
CANCELLED = "已取消"
class SparkApplicationType(Enum):
"""Spark应用类型"""
BATCH = "批处理"
STREAMING = "流处理"
MACHINE_LEARNING = "机器学习"
GRAPH_PROCESSING = "图处理"
SQL = "SQL查询"
@dataclass
class SparkJob:
"""Spark作业"""
job_id: str
app_id: str
name: str
job_type: SparkApplicationType
status: SparkJobStatus
submit_time: datetime
start_time: Optional[datetime]
end_time: Optional[datetime]
driver_memory: str
executor_memory: str
executor_cores: int
num_executors: int
input_data_size: float # GB
output_data_size: float # GB
stages: List[Dict[str, Any]]
metrics: Dict[str, Any]
@dataclass
class SparkCluster:
"""Spark集群"""
cluster_id: str
master_url: str
total_cores: int
total_memory_gb: float
worker_nodes: List[str]
active_applications: List[str]
completed_applications: List[str]
class SparkClusterManager:
"""
Spark集群管理器
"""
def __init__(self):
self.clusters = {}
self.applications = {}
self.jobs = {}
self.performance_metrics = {}
self.resource_usage = {}
def create_cluster(self, cluster_config: Dict[str, Any]) -> Dict[str, Any]:
"""
创建Spark集群
Args:
cluster_config: 集群配置
Returns:
Dict[str, Any]: 创建结果
"""
cluster = SparkCluster(
cluster_id=cluster_config['cluster_id'],
master_url=cluster_config.get('master_url', 'spark://master:7077'),
total_cores=cluster_config.get('total_cores', 16),
total_memory_gb=cluster_config.get('total_memory_gb', 64.0),
worker_nodes=cluster_config.get('worker_nodes', ['worker1', 'worker2', 'worker3']),
active_applications=[],
completed_applications=[]
)
self.clusters[cluster.cluster_id] = cluster
# 初始化资源使用情况
self.resource_usage[cluster.cluster_id] = {
'used_cores': 0,
'used_memory_gb': 0.0,
'cpu_utilization': 0.0,
'memory_utilization': 0.0
}
return {
'status': 'success',
'cluster_id': cluster.cluster_id,
'master_url': cluster.master_url,
'total_cores': cluster.total_cores,
'total_memory_gb': cluster.total_memory_gb,
'worker_nodes': cluster.worker_nodes
}
def submit_application(self, app_config: Dict[str, Any]) -> Dict[str, Any]:
"""
提交Spark应用
Args:
app_config: 应用配置
Returns:
Dict[str, Any]: 提交结果
"""
cluster_id = app_config.get('cluster_id')
if cluster_id not in self.clusters:
return {'status': 'error', 'message': f'Cluster {cluster_id} not found'}
cluster = self.clusters[cluster_id]
# 检查资源是否足够
required_cores = app_config.get('executor_cores', 2) * app_config.get('num_executors', 2)
required_memory = float(app_config.get('executor_memory', '2g').replace('g', '')) * app_config.get('num_executors', 2)
current_usage = self.resource_usage[cluster_id]
if (current_usage['used_cores'] + required_cores > cluster.total_cores or
current_usage['used_memory_gb'] + required_memory > cluster.total_memory_gb):
return {
'status': 'error',
'message': 'Insufficient resources',
'required': {'cores': required_cores, 'memory_gb': required_memory},
'available': {
'cores': cluster.total_cores - current_usage['used_cores'],
'memory_gb': cluster.total_memory_gb - current_usage['used_memory_gb']
}
}
# 创建应用
app_id = f"app-{datetime.now().strftime('%Y%m%d-%H%M%S')}-{len(self.applications)}"
application = {
'app_id': app_id,
'name': app_config.get('name', 'Spark Application'),
'cluster_id': cluster_id,
'app_type': SparkApplicationType(app_config.get('app_type', 'BATCH')),
'status': SparkJobStatus.SUBMITTED,
'submit_time': datetime.now(),
'driver_memory': app_config.get('driver_memory', '1g'),
'executor_memory': app_config.get('executor_memory', '2g'),
'executor_cores': app_config.get('executor_cores', 2),
'num_executors': app_config.get('num_executors', 2),
'main_class': app_config.get('main_class'),
'jar_file': app_config.get('jar_file'),
'arguments': app_config.get('arguments', []),
'configuration': app_config.get('configuration', {})
}
self.applications[app_id] = application
cluster.active_applications.append(app_id)
# 更新资源使用
current_usage['used_cores'] += required_cores
current_usage['used_memory_gb'] += required_memory
current_usage['cpu_utilization'] = (current_usage['used_cores'] / cluster.total_cores) * 100
current_usage['memory_utilization'] = (current_usage['used_memory_gb'] / cluster.total_memory_gb) * 100
return {
'status': 'success',
'app_id': app_id,
'cluster_id': cluster_id,
'submit_time': application['submit_time'].isoformat(),
'resource_allocation': {
'cores': required_cores,
'memory_gb': required_memory,
'executors': application['num_executors']
}
}
def create_spark_job(self, job_config: Dict[str, Any]) -> Dict[str, Any]:
"""
创建Spark作业
Args:
job_config: 作业配置
Returns:
Dict[str, Any]: 创建结果
"""
app_id = job_config.get('app_id')
if app_id not in self.applications:
return {'status': 'error', 'message': f'Application {app_id} not found'}
application = self.applications[app_id]
job = SparkJob(
job_id=f"job-{len(self.jobs)}",
app_id=app_id,
name=job_config.get('name', 'Spark Job'),
job_type=application['app_type'],
status=SparkJobStatus.SUBMITTED,
submit_time=datetime.now(),
start_time=None,
end_time=None,
driver_memory=application['driver_memory'],
executor_memory=application['executor_memory'],
executor_cores=application['executor_cores'],
num_executors=application['num_executors'],
input_data_size=job_config.get('input_data_size', 1.0),
output_data_size=0.0,
stages=[],
metrics={}
)
self.jobs[job.job_id] = job
return {
'status': 'success',
'job_id': job.job_id,
'app_id': app_id,
'submit_time': job.submit_time.isoformat()
}
def start_job(self, job_id: str) -> Dict[str, Any]:
"""
启动作业
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 启动结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
job.status = SparkJobStatus.RUNNING
job.start_time = datetime.now()
# 模拟创建stages
stages = self._create_job_stages(job)
job.stages = stages
return {
'status': 'success',
'job_id': job_id,
'start_time': job.start_time.isoformat(),
'stages': len(stages),
'estimated_duration': self._estimate_job_duration(job)
}
def _create_job_stages(self, job: SparkJob) -> List[Dict[str, Any]]:
"""
创建作业阶段
Args:
job: Spark作业
Returns:
List[Dict[str, Any]]: 阶段列表
"""
stages = []
if job.job_type == SparkApplicationType.BATCH:
# 批处理作业的典型阶段
stages = [
{
'stage_id': 0,
'name': 'Data Loading',
'num_tasks': job.num_executors * 2,
'status': 'pending',
'input_size_gb': job.input_data_size,
'output_size_gb': job.input_data_size * 0.9
},
{
'stage_id': 1,
'name': 'Data Transformation',
'num_tasks': job.num_executors * 4,
'status': 'pending',
'input_size_gb': job.input_data_size * 0.9,
'output_size_gb': job.input_data_size * 0.7
},
{
'stage_id': 2,
'name': 'Data Aggregation',
'num_tasks': job.num_executors,
'status': 'pending',
'input_size_gb': job.input_data_size * 0.7,
'output_size_gb': job.input_data_size * 0.3
}
]
elif job.job_type == SparkApplicationType.MACHINE_LEARNING:
# 机器学习作业的典型阶段
stages = [
{
'stage_id': 0,
'name': 'Feature Extraction',
'num_tasks': job.num_executors * 2,
'status': 'pending',
'input_size_gb': job.input_data_size,
'output_size_gb': job.input_data_size * 1.2
},
{
'stage_id': 1,
'name': 'Model Training',
'num_tasks': job.num_executors * 8,
'status': 'pending',
'input_size_gb': job.input_data_size * 1.2,
'output_size_gb': 0.1
},
{
'stage_id': 2,
'name': 'Model Evaluation',
'num_tasks': job.num_executors,
'status': 'pending',
'input_size_gb': job.input_data_size * 0.2,
'output_size_gb': 0.01
}
]
return stages
def _estimate_job_duration(self, job: SparkJob) -> str:
"""
估算作业持续时间
Args:
job: Spark作业
Returns:
str: 估算时间
"""
# 基于数据大小和资源的简单估算
base_time = job.input_data_size * 60 # 每GB 60秒
resource_factor = max(1, 8 / job.num_executors) # 资源调整因子
estimated_seconds = base_time * resource_factor
if estimated_seconds < 60:
return f"{int(estimated_seconds)}秒"
elif estimated_seconds < 3600:
return f"{int(estimated_seconds/60)}分钟"
else:
return f"{estimated_seconds/3600:.1f}小时"
def complete_job(self, job_id: str, success: bool = True) -> Dict[str, Any]:
"""
完成作业
Args:
job_id: 作业ID
success: 是否成功
Returns:
Dict[str, Any]: 完成结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
job.status = SparkJobStatus.SUCCEEDED if success else SparkJobStatus.FAILED
job.end_time = datetime.now()
# 计算作业指标
if job.start_time:
duration = (job.end_time - job.start_time).total_seconds()
# 模拟输出数据大小
if success:
job.output_data_size = sum(stage.get('output_size_gb', 0) for stage in job.stages)
job.metrics = {
'duration_seconds': duration,
'input_data_gb': job.input_data_size,
'output_data_gb': job.output_data_size,
'data_processing_rate_gb_per_sec': job.input_data_size / duration if duration > 0 else 0,
'total_tasks': sum(stage.get('num_tasks', 0) for stage in job.stages),
'resource_hours': (job.num_executors * job.executor_cores * duration) / 3600
}
# 释放资源
self._release_job_resources(job)
return {
'status': 'success',
'job_id': job_id,
'final_status': job.status.value,
'end_time': job.end_time.isoformat(),
'metrics': job.metrics
}
def _release_job_resources(self, job: SparkJob):
"""
释放作业资源
Args:
job: Spark作业
"""
application = self.applications.get(job.app_id)
if not application:
return
cluster_id = application['cluster_id']
if cluster_id not in self.resource_usage:
return
# 计算释放的资源
released_cores = job.executor_cores * job.num_executors
released_memory = float(job.executor_memory.replace('g', '')) * job.num_executors
# 更新资源使用
current_usage = self.resource_usage[cluster_id]
current_usage['used_cores'] = max(0, current_usage['used_cores'] - released_cores)
current_usage['used_memory_gb'] = max(0, current_usage['used_memory_gb'] - released_memory)
cluster = self.clusters[cluster_id]
current_usage['cpu_utilization'] = (current_usage['used_cores'] / cluster.total_cores) * 100
current_usage['memory_utilization'] = (current_usage['used_memory_gb'] / cluster.total_memory_gb) * 100
def get_job_status(self, job_id: str) -> Dict[str, Any]:
"""
获取作业状态
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业状态
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
result = {
'job_id': job.job_id,
'app_id': job.app_id,
'name': job.name,
'job_type': job.job_type.value,
'status': job.status.value,
'submit_time': job.submit_time.isoformat(),
'resource_allocation': {
'driver_memory': job.driver_memory,
'executor_memory': job.executor_memory,
'executor_cores': job.executor_cores,
'num_executors': job.num_executors
},
'data_info': {
'input_size_gb': job.input_data_size,
'output_size_gb': job.output_data_size
},
'stages': job.stages
}
if job.start_time:
result['start_time'] = job.start_time.isoformat()
if job.end_time:
result['end_time'] = job.end_time.isoformat()
if job.metrics:
result['metrics'] = job.metrics
return result
def get_cluster_status(self, cluster_id: str) -> Dict[str, Any]:
"""
获取集群状态
Args:
cluster_id: 集群ID
Returns:
Dict[str, Any]: 集群状态
"""
if cluster_id not in self.clusters:
return {'status': 'error', 'message': f'Cluster {cluster_id} not found'}
cluster = self.clusters[cluster_id]
usage = self.resource_usage[cluster_id]
# 统计应用状态
active_apps = []
for app_id in cluster.active_applications:
if app_id in self.applications:
app = self.applications[app_id]
active_apps.append({
'app_id': app_id,
'name': app['name'],
'type': app['app_type'].value,
'status': app['status'].value,
'submit_time': app['submit_time'].isoformat()
})
return {
'cluster_id': cluster_id,
'master_url': cluster.master_url,
'resources': {
'total_cores': cluster.total_cores,
'total_memory_gb': cluster.total_memory_gb,
'used_cores': usage['used_cores'],
'used_memory_gb': usage['used_memory_gb'],
'available_cores': cluster.total_cores - usage['used_cores'],
'available_memory_gb': cluster.total_memory_gb - usage['used_memory_gb'],
'cpu_utilization': round(usage['cpu_utilization'], 2),
'memory_utilization': round(usage['memory_utilization'], 2)
},
'worker_nodes': cluster.worker_nodes,
'applications': {
'active_count': len(cluster.active_applications),
'completed_count': len(cluster.completed_applications),
'active_applications': active_apps
}
}
def optimize_resource_allocation(self, cluster_id: str) -> Dict[str, Any]:
"""
优化资源分配
Args:
cluster_id: 集群ID
Returns:
Dict[str, Any]: 优化建议
"""
if cluster_id not in self.clusters:
return {'status': 'error', 'message': f'Cluster {cluster_id} not found'}
cluster = self.clusters[cluster_id]
usage = self.resource_usage[cluster_id]
recommendations = []
# CPU利用率分析
if usage['cpu_utilization'] > 90:
recommendations.append({
'type': 'resource_shortage',
'priority': 'high',
'message': 'CPU利用率过高，建议增加worker节点或减少并发作业',
'suggested_action': 'scale_out'
})
elif usage['cpu_utilization'] < 30:
recommendations.append({
'type': 'resource_waste',
'priority': 'medium',
'message': 'CPU利用率较低，可以考虑减少资源或增加作业并发度',
'suggested_action': 'scale_in_or_increase_concurrency'
})
# 内存利用率分析
if usage['memory_utilization'] > 85:
recommendations.append({
'type': 'memory_pressure',
'priority': 'high',
'message': '内存使用率过高，可能导致GC压力，建议优化内存配置',
'suggested_action': 'optimize_memory_config'
})
# 负载均衡分析
active_apps = len(cluster.active_applications)
worker_count = len(cluster.worker_nodes)
if active_apps > worker_count * 2:
recommendations.append({
'type': 'load_imbalance',
'priority': 'medium',
'message': '作业数量较多，建议检查负载均衡配置',
'suggested_action': 'check_load_balancing'
})
# 配置优化建议
optimization_suggestions = {
'executor_memory': self._suggest_executor_memory(cluster, usage),
'executor_cores': self._suggest_executor_cores(cluster, usage),
'dynamic_allocation': usage['cpu_utilization'] < 50,
'speculation': usage['cpu_utilization'] > 70
}
return {
'cluster_id': cluster_id,
'current_utilization': {
'cpu': usage['cpu_utilization'],
'memory': usage['memory_utilization']
},
'recommendations': recommendations,
'optimization_suggestions': optimization_suggestions,
'analysis_time': datetime.now().isoformat()
}
def _suggest_executor_memory(self, cluster: SparkCluster, usage: Dict[str, Any]) -> str:
"""
建议executor内存配置
Args:
cluster: Spark集群
usage: 资源使用情况
Returns:
str: 建议的内存配置
"""
# 基于集群总内存和worker数量的简单计算
memory_per_worker = cluster.total_memory_gb / len(cluster.worker_nodes)
suggested_memory = max(1, int(memory_per_worker * 0.8 / 2)) # 保留20%给系统，每个worker运行2个executor
return f"{suggested_memory}g"
def _suggest_executor_cores(self, cluster: SparkCluster, usage: Dict[str, Any]) -> int:
"""
建议executor核心数配置
Args:
cluster: Spark集群
usage: 资源使用情况
Returns:
int: 建议的核心数
"""
# 基于集群总核心数和worker数量的简单计算
cores_per_worker = cluster.total_cores / len(cluster.worker_nodes)
suggested_cores = max(1, int(cores_per_worker / 2)) # 每个worker运行2个executor
return min(suggested_cores, 5) # 限制最大核心数为5
# 使用示例
if __name__ == "__main__":
# 创建Spark集群管理器
spark_manager = SparkClusterManager()
print("=== Spark集群管理示例 ===")
# 创建集群
print("\n=== 创建集群 ===")
cluster_config = {
'cluster_id': 'spark-cluster-1',
'master_url': 'spark://master:7077',
'total_cores': 32,
'total_memory_gb': 128.0,
'worker_nodes': ['worker1', 'worker2', 'worker3', 'worker4']
}
cluster_result = spark_manager.create_cluster(cluster_config)
print(f"集群创建结果: {cluster_result}")
# 提交应用
print("\n=== 提交应用 ===")
app_config = {
'cluster_id': 'spark-cluster-1',
'name': 'Data Processing Job',
'app_type': 'BATCH',
'driver_memory': '2g',
'executor_memory': '4g',
'executor_cores': 2,
'num_executors': 4,
'main_class': 'com.example.DataProcessor',
'jar_file': '/path/to/app.jar'
}
app_result = spark_manager.submit_application(app_config)
print(f"应用提交结果: {app_result}")
if app_result['status'] == 'success':
app_id = app_result['app_id']
# 创建作业
print("\n=== 创建作业 ===")
job_config = {
'app_id': app_id,
'name': 'ETL Job',
'input_data_size': 10.0 # 10GB
}
job_result = spark_manager.create_spark_job(job_config)
print(f"作业创建结果: {job_result}")
if job_result['status'] == 'success':
job_id = job_result['job_id']
# 启动作业
print("\n=== 启动作业 ===")
start_result = spark_manager.start_job(job_id)
print(f"作业启动结果: {start_result}")
# 获取作业状态
print("\n=== 作业状态 ===")
job_status = spark_manager.get_job_status(job_id)
print(f"作业状态: {job_status['status']}")
print(f"阶段数: {len(job_status['stages'])}")
print(f"预估时间: {start_result.get('estimated_duration')}")
# 完成作业
print("\n=== 完成作业 ===")
complete_result = spark_manager.complete_job(job_id, success=True)
print(f"作业完成结果: {complete_result}")
# 获取集群状态
print("\n=== 集群状态 ===")
cluster_status = spark_manager.get_cluster_status('spark-cluster-1')
print(f"集群ID: {cluster_status['cluster_id']}")
print(f"CPU利用率: {cluster_status['resources']['cpu_utilization']}%")
print(f"内存利用率: {cluster_status['resources']['memory_utilization']}%")
print(f"活跃应用数: {cluster_status['applications']['active_count']}")
# 资源优化建议
print("\n=== 资源优化建议 ===")
optimization = spark_manager.optimize_resource_allocation('spark-cluster-1')
print(f"当前CPU利用率: {optimization['current_utilization']['cpu']}%")
print(f"当前内存利用率: {optimization['current_utilization']['memory']}%")
print("\n优化建议:")
for rec in optimization['recommendations']:
print(f" - {rec['type']} ({rec['priority']}): {rec['message']}")
print("\n配置建议:")
suggestions = optimization['optimization_suggestions']
print(f" - 建议executor内存: {suggestions['executor_memory']}")
print(f" - 建议executor核心数: {suggestions['executor_cores']}")
print(f" - 启用动态分配: {suggestions['dynamic_allocation']}")
print(f" - 启用推测执行: {suggestions['speculation']}")
3.2 Apache Flink详解
from typing import Dict, List, Any, Optional, Callable, Union
from dataclasses import dataclass
from enum import Enum
from datetime import datetime, timedelta
import json
import time
class FlinkJobStatus(Enum):
"""Flink作业状态"""
CREATED = "已创建"
RUNNING = "运行中"
FINISHED = "已完成"
CANCELED = "已取消"
FAILED = "失败"
RESTARTING = "重启中"
class FlinkJobType(Enum):
"""Flink作业类型"""
STREAMING = "流处理"
BATCH = "批处理"
CEP = "复杂事件处理"
MACHINE_LEARNING = "机器学习"
GRAPH = "图处理"
class CheckpointStatus(Enum):
"""检查点状态"""
COMPLETED = "已完成"
IN_PROGRESS = "进行中"
FAILED = "失败"
@dataclass
class FlinkCheckpoint:
"""Flink检查点"""
checkpoint_id: int
timestamp: datetime
status: CheckpointStatus
size_bytes: int
duration_ms: int
external_path: Optional[str] = None
@dataclass
class FlinkTaskManager:
"""Flink任务管理器"""
tm_id: str
host: str
port: int
slots_total: int
slots_available: int
memory_mb: int
cpu_cores: int
network_memory_mb: int
managed_memory_mb: int
@dataclass
class FlinkJob:
"""Flink作业"""
job_id: str
name: str
job_type: FlinkJobType
status: FlinkJobStatus
start_time: datetime
end_time: Optional[datetime]
parallelism: int
max_parallelism: int
checkpointing_enabled: bool
checkpoint_interval_ms: int
savepoint_path: Optional[str]
restart_strategy: str
metrics: Dict[str, Any]
checkpoints: List[FlinkCheckpoint]
class FlinkClusterManager:
"""
Flink集群管理器
"""
def __init__(self):
self.job_manager_host = "localhost"
self.job_manager_port = 8081
self.task_managers = {}
self.jobs = {}
self.checkpoints = {}
self.savepoints = {}
self.cluster_metrics = {
'total_slots': 0,
'available_slots': 0,
'running_jobs': 0,
'completed_jobs': 0,
'failed_jobs': 0
}
def add_task_manager(self, tm_config: Dict[str, Any]) -> Dict[str, Any]:
"""
添加任务管理器
Args:
tm_config: 任务管理器配置
Returns:
Dict[str, Any]: 添加结果
"""
tm = FlinkTaskManager(
tm_id=tm_config['tm_id'],
host=tm_config.get('host', 'localhost'),
port=tm_config.get('port', 6121),
slots_total=tm_config.get('slots_total', 4),
slots_available=tm_config.get('slots_total', 4),
memory_mb=tm_config.get('memory_mb', 4096),
cpu_cores=tm_config.get('cpu_cores', 4),
network_memory_mb=tm_config.get('network_memory_mb', 512),
managed_memory_mb=tm_config.get('managed_memory_mb', 1024)
)
self.task_managers[tm.tm_id] = tm
# 更新集群指标
self.cluster_metrics['total_slots'] += tm.slots_total
self.cluster_metrics['available_slots'] += tm.slots_available
return {
'status': 'success',
'tm_id': tm.tm_id,
'host': tm.host,
'slots': tm.slots_total,
'memory_mb': tm.memory_mb
}
def submit_job(self, job_config: Dict[str, Any]) -> Dict[str, Any]:
"""
提交Flink作业
Args:
job_config: 作业配置
Returns:
Dict[str, Any]: 提交结果
"""
# 检查资源是否足够
required_slots = job_config.get('parallelism', 1)
if self.cluster_metrics['available_slots'] < required_slots:
return {
'status': 'error',
'message': f'Insufficient slots. Required: {required_slots}, Available: {self.cluster_metrics["available_slots"]}'
}
# 创建作业
job_id = f"job-{datetime.now().strftime('%Y%m%d-%H%M%S')}-{len(self.jobs)}"
job = FlinkJob(
job_id=job_id,
name=job_config.get('name', 'Flink Job'),
job_type=FlinkJobType(job_config.get('job_type', 'STREAMING')),
status=FlinkJobStatus.CREATED,
start_time=datetime.now(),
end_time=None,
parallelism=job_config.get('parallelism', 1),
max_parallelism=job_config.get('max_parallelism', 128),
checkpointing_enabled=job_config.get('checkpointing_enabled', True),
checkpoint_interval_ms=job_config.get('checkpoint_interval_ms', 60000),
savepoint_path=job_config.get('savepoint_path'),
restart_strategy=job_config.get('restart_strategy', 'fixed-delay'),
metrics={},
checkpoints=[]
)
self.jobs[job_id] = job
# 分配资源
self._allocate_slots(job)
return {
'status': 'success',
'job_id': job_id,
'name': job.name,
'parallelism': job.parallelism,
'submit_time': job.start_time.isoformat()
}
def _allocate_slots(self, job: FlinkJob):
"""
为作业分配slot
Args:
job: Flink作业
"""
slots_needed = job.parallelism
allocated = 0
for tm in self.task_managers.values():
if allocated >= slots_needed:
break
slots_to_allocate = min(tm.slots_available, slots_needed - allocated)
tm.slots_available -= slots_to_allocate
allocated += slots_to_allocate
# 更新集群指标
self.cluster_metrics['available_slots'] -= slots_needed
def start_job(self, job_id: str) -> Dict[str, Any]:
"""
启动作业
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 启动结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
job.status = FlinkJobStatus.RUNNING
# 更新集群指标
self.cluster_metrics['running_jobs'] += 1
# 如果启用检查点，创建初始检查点
if job.checkpointing_enabled:
self._create_checkpoint(job)
return {
'status': 'success',
'job_id': job_id,
'status_change': f'{FlinkJobStatus.CREATED.value} -> {FlinkJobStatus.RUNNING.value}',
'start_time': job.start_time.isoformat()
}
def _create_checkpoint(self, job: FlinkJob) -> FlinkCheckpoint:
"""
创建检查点
Args:
job: Flink作业
Returns:
FlinkCheckpoint: 检查点
"""
checkpoint_id = len(job.checkpoints)
checkpoint = FlinkCheckpoint(
checkpoint_id=checkpoint_id,
timestamp=datetime.now(),
status=CheckpointStatus.IN_PROGRESS,
size_bytes=0,
duration_ms=0
)
# 模拟检查点完成
checkpoint.status = CheckpointStatus.COMPLETED
checkpoint.size_bytes = job.parallelism * 1024 * 1024 # 每个并行度1MB
checkpoint.duration_ms = job.parallelism * 100 # 每个并行度100ms
checkpoint.external_path = f"/checkpoints/{job.job_id}/chk-{checkpoint_id}"
job.checkpoints.append(checkpoint)
return checkpoint
def create_savepoint(self, job_id: str, target_directory: Optional[str] = None) -> Dict[str, Any]:
"""
创建保存点
Args:
job_id: 作业ID
target_directory: 目标目录
Returns:
Dict[str, Any]: 创建结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
if job.status != FlinkJobStatus.RUNNING:
return {'status': 'error', 'message': f'Job {job_id} is not running'}
savepoint_id = f"savepoint-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
savepoint_path = target_directory or f"/savepoints/{job_id}/{savepoint_id}"
savepoint = {
'savepoint_id': savepoint_id,
'job_id': job_id,
'path': savepoint_path,
'timestamp': datetime.now(),
'size_bytes': job.parallelism * 2 * 1024 * 1024, # 每个并行度2MB
'trigger_type': 'manual'
}
self.savepoints[savepoint_id] = savepoint
job.savepoint_path = savepoint_path
return {
'status': 'success',
'savepoint_id': savepoint_id,
'path': savepoint_path,
'size_bytes': savepoint['size_bytes'],
'timestamp': savepoint['timestamp'].isoformat()
}
def stop_job(self, job_id: str, with_savepoint: bool = False,
target_directory: Optional[str] = None) -> Dict[str, Any]:
"""
停止作业
Args:
job_id: 作业ID
with_savepoint: 是否创建保存点
target_directory: 保存点目录
Returns:
Dict[str, Any]: 停止结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
result = {
'status': 'success',
'job_id': job_id,
'previous_status': job.status.value
}
# 如果需要创建保存点
if with_savepoint and job.status == FlinkJobStatus.RUNNING:
savepoint_result = self.create_savepoint(job_id, target_directory)
if savepoint_result['status'] == 'success':
result['savepoint'] = savepoint_result
else:
return savepoint_result
# 停止作业
job.status = FlinkJobStatus.FINISHED
job.end_time = datetime.now()
# 释放资源
self._release_slots(job)
# 更新集群指标
if result['previous_status'] == FlinkJobStatus.RUNNING.value:
self.cluster_metrics['running_jobs'] -= 1
self.cluster_metrics['completed_jobs'] += 1
result['end_time'] = job.end_time.isoformat()
return result
def _release_slots(self, job: FlinkJob):
"""
释放作业占用的slot
Args:
job: Flink作业
"""
slots_to_release = job.parallelism
released = 0
for tm in self.task_managers.values():
if released >= slots_to_release:
break
slots_can_release = min(tm.slots_total - tm.slots_available, slots_to_release - released)
tm.slots_available += slots_can_release
released += slots_can_release
# 更新集群指标
self.cluster_metrics['available_slots'] += slots_to_release
def cancel_job(self, job_id: str) -> Dict[str, Any]:
"""
取消作业
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 取消结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
previous_status = job.status
job.status = FlinkJobStatus.CANCELED
job.end_time = datetime.now()
# 释放资源
self._release_slots(job)
# 更新集群指标
if previous_status == FlinkJobStatus.RUNNING:
self.cluster_metrics['running_jobs'] -= 1
self.cluster_metrics['failed_jobs'] += 1
return {
'status': 'success',
'job_id': job_id,
'previous_status': previous_status.value,
'current_status': job.status.value,
'end_time': job.end_time.isoformat()
}
def get_job_details(self, job_id: str) -> Dict[str, Any]:
"""
获取作业详情
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业详情
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
# 计算运行时间
if job.end_time:
duration = job.end_time - job.start_time
else:
duration = datetime.now() - job.start_time
# 获取最新检查点信息
latest_checkpoint = None
if job.checkpoints:
latest_checkpoint = {
'checkpoint_id': job.checkpoints[-1].checkpoint_id,
'timestamp': job.checkpoints[-1].timestamp.isoformat(),
'status': job.checkpoints[-1].status.value,
'size_bytes': job.checkpoints[-1].size_bytes,
'duration_ms': job.checkpoints[-1].duration_ms
}
result = {
'job_id': job.job_id,
'name': job.name,
'job_type': job.job_type.value,
'status': job.status.value,
'start_time': job.start_time.isoformat(),
'duration_seconds': int(duration.total_seconds()),
'parallelism': job.parallelism,
'max_parallelism': job.max_parallelism,
'checkpointing': {
'enabled': job.checkpointing_enabled,
'interval_ms': job.checkpoint_interval_ms,
'total_checkpoints': len(job.checkpoints),
'latest_checkpoint': latest_checkpoint
},
'restart_strategy': job.restart_strategy
}
if job.end_time:
result['end_time'] = job.end_time.isoformat()
if job.savepoint_path:
result['savepoint_path'] = job.savepoint_path
return result
def get_cluster_overview(self) -> Dict[str, Any]:
"""
获取集群概览
Returns:
Dict[str, Any]: 集群概览
"""
# 统计任务管理器信息
tm_summary = {
'total_task_managers': len(self.task_managers),
'total_memory_mb': sum(tm.memory_mb for tm in self.task_managers.values()),
'total_cpu_cores': sum(tm.cpu_cores for tm in self.task_managers.values()),
'task_managers': []
}
for tm in self.task_managers.values():
tm_summary['task_managers'].append({
'tm_id': tm.tm_id,
'host': tm.host,
'slots_total': tm.slots_total,
'slots_available': tm.slots_available,
'slots_used': tm.slots_total - tm.slots_available,
'memory_mb': tm.memory_mb,
'cpu_cores': tm.cpu_cores
})
# 统计作业信息
job_summary = {
'total_jobs': len(self.jobs),
'running_jobs': self.cluster_metrics['running_jobs'],
'completed_jobs': self.cluster_metrics['completed_jobs'],
'failed_jobs': self.cluster_metrics['failed_jobs'],
'jobs_by_type': {}
}
# 按类型统计作业
for job in self.jobs.values():
job_type = job.job_type.value
if job_type not in job_summary['jobs_by_type']:
job_summary['jobs_by_type'][job_type] = 0
job_summary['jobs_by_type'][job_type] += 1
return {
'job_manager': {
'host': self.job_manager_host,
'port': self.job_manager_port
},
'cluster_metrics': self.cluster_metrics,
'task_managers': tm_summary,
'jobs': job_summary,
'timestamp': datetime.now().isoformat()
}
def get_job_metrics(self, job_id: str) -> Dict[str, Any]:
"""
获取作业指标
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业指标
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
# 模拟一些指标数据
if job.status == FlinkJobStatus.RUNNING:
# 运行中的作业有实时指标
metrics = {
'records_in_per_second': job.parallelism * 1000,
'records_out_per_second': job.parallelism * 950,
'bytes_in_per_second': job.parallelism * 1024 * 100,
'bytes_out_per_second': job.parallelism * 1024 * 95,
'backpressure_ratio': 0.1,
'cpu_utilization': 0.75,
'memory_utilization': 0.65,
'gc_time_ms_per_second': 50,
'checkpoint_duration_ms': 500,
'checkpoint_size_bytes': job.parallelism * 1024 * 1024
}
else:
# 非运行状态的作业只有历史指标
duration = (job.end_time - job.start_time).total_seconds() if job.end_time else 0
metrics = {
'total_records_processed': int(job.parallelism * 1000 * duration),
'total_bytes_processed': int(job.parallelism * 1024 * 100 * duration),
'average_processing_rate': job.parallelism * 1000 if duration > 0 else 0,
'total_checkpoints': len(job.checkpoints),
'successful_checkpoints': len([cp for cp in job.checkpoints if cp.status == CheckpointStatus.COMPLETED]),
'failed_checkpoints': len([cp for cp in job.checkpoints if cp.status == CheckpointStatus.FAILED])
}
return {
'job_id': job_id,
'status': job.status.value,
'metrics': metrics,
'timestamp': datetime.now().isoformat()
}
def restart_job(self, job_id: str, from_savepoint: Optional[str] = None) -> Dict[str, Any]:
"""
重启作业
Args:
job_id: 作业ID
from_savepoint: 从保存点恢复
Returns:
Dict[str, Any]: 重启结果
"""
if job_id not in self.jobs:
return {'status': 'error', 'message': f'Job {job_id} not found'}
job = self.jobs[job_id]
# 检查作业状态
if job.status == FlinkJobStatus.RUNNING:
return {'status': 'error', 'message': f'Job {job_id} is already running'}
# 检查资源
if self.cluster_metrics['available_slots'] < job.parallelism:
return {
'status': 'error',
'message': f'Insufficient slots. Required: {job.parallelism}, Available: {self.cluster_metrics["available_slots"]}'
}
# 重启作业
job.status = FlinkJobStatus.RESTARTING
# 分配资源
self._allocate_slots(job)
# 更新状态为运行中
job.status = FlinkJobStatus.RUNNING
job.start_time = datetime.now()
job.end_time = None
# 更新集群指标
self.cluster_metrics['running_jobs'] += 1
if job.status in [FlinkJobStatus.FAILED, FlinkJobStatus.CANCELED]:
self.cluster_metrics['failed_jobs'] -= 1
result = {
'status': 'success',
'job_id': job_id,
'restart_time': job.start_time.isoformat(),
'parallelism': job.parallelism
}
if from_savepoint:
result['restored_from_savepoint'] = from_savepoint
return result
# 使用示例
if __name__ == "__main__":
# 创建Flink集群管理器
flink_manager = FlinkClusterManager()
print("=== Flink集群管理示例 ===")
# 添加任务管理器
print("\n=== 添加任务管理器 ===")
for i in range(3):
tm_config = {
'tm_id': f'taskmanager-{i+1}',
'host': f'worker{i+1}',
'port': 6121 + i,
'slots_total': 4,
'memory_mb': 4096,
'cpu_cores': 4
}
tm_result = flink_manager.add_task_manager(tm_config)
print(f"任务管理器 {i+1}: {tm_result}")
# 提交流处理作业
print("\n=== 提交流处理作业 ===")
streaming_job_config = {
'name': 'Real-time Analytics',
'job_type': 'STREAMING',
'parallelism': 6,
'max_parallelism': 128,
'checkpointing_enabled': True,
'checkpoint_interval_ms': 30000,
'restart_strategy': 'fixed-delay'
}
job_result = flink_manager.submit_job(streaming_job_config)
print(f"作业提交结果: {job_result}")
if job_result['status'] == 'success':
job_id = job_result['job_id']
# 启动作业
print("\n=== 启动作业 ===")
start_result = flink_manager.start_job(job_id)
print(f"作业启动结果: {start_result}")
# 获取作业详情
print("\n=== 作业详情 ===")
job_details = flink_manager.get_job_details(job_id)
print(f"作业名称: {job_details['name']}")
print(f"作业状态: {job_details['status']}")
print(f"并行度: {job_details['parallelism']}")
print(f"检查点: 启用={job_details['checkpointing']['enabled']}, 间隔={job_details['checkpointing']['interval_ms']}ms")
# 获取作业指标
print("\n=== 作业指标 ===")
metrics = flink_manager.get_job_metrics(job_id)
print(f"输入速率: {metrics['metrics']['records_in_per_second']} records/sec")
print(f"输出速率: {metrics['metrics']['records_out_per_second']} records/sec")
print(f"CPU利用率: {metrics['metrics']['cpu_utilization']*100:.1f}%")
print(f"内存利用率: {metrics['metrics']['memory_utilization']*100:.1f}%")
# 创建保存点
print("\n=== 创建保存点 ===")
savepoint_result = flink_manager.create_savepoint(job_id)
print(f"保存点创建结果: {savepoint_result}")
# 停止作业（带保存点）
print("\n=== 停止作业 ===")
stop_result = flink_manager.stop_job(job_id, with_savepoint=True)
print(f"作业停止结果: {stop_result}")
# 重启作业
print("\n=== 重启作业 ===")
restart_result = flink_manager.restart_job(job_id)
print(f"作业重启结果: {restart_result}")
# 获取集群概览
print("\n=== 集群概览 ===")
cluster_overview = flink_manager.get_cluster_overview()
print(f"任务管理器数量: {cluster_overview['task_managers']['total_task_managers']}")
print(f"总slot数: {cluster_overview['cluster_metrics']['total_slots']}")
print(f"可用slot数: {cluster_overview['cluster_metrics']['available_slots']}")
print(f"运行中作业: {cluster_overview['cluster_metrics']['running_jobs']}")
print(f"已完成作业: {cluster_overview['cluster_metrics']['completed_jobs']}")
print("\n任务管理器详情:")
for tm in cluster_overview['task_managers']['task_managers']:
print(f" {tm['tm_id']}: {tm['slots_used']}/{tm['slots_total']} slots used, {tm['memory_mb']}MB memory")
4. 数据查询组件
4.1 Apache Hive详解
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass
from enum import Enum
from datetime import datetime
import re
import json
class HiveTableType(Enum):
"""Hive表类型"""
MANAGED = "托管表"
EXTERNAL = "外部表"
TEMPORARY = "临时表"
VIEW = "视图"
class HiveDataType(Enum):
"""Hive数据类型"""
STRING = "string"
INT = "int"
BIGINT = "bigint"
DOUBLE = "double"
BOOLEAN = "boolean"
TIMESTAMP = "timestamp"
DATE = "date"
ARRAY = "array"
MAP = "map"
STRUCT = "struct"
class HiveQueryStatus(Enum):
"""Hive查询状态"""
PENDING = "等待中"
RUNNING = "运行中"
FINISHED = "已完成"
ERROR = "错误"
KILLED = "已终止"
@dataclass
class HiveColumn:
"""Hive列定义"""
name: str
data_type: HiveDataType
comment: Optional[str] = None
nullable: bool = True
@dataclass
class HivePartition:
"""Hive分区定义"""
column: str
value: str
location: Optional[str] = None
@dataclass
class HiveTable:
"""Hive表定义"""
database: str
table_name: str
table_type: HiveTableType
columns: List[HiveColumn]
partition_columns: List[HiveColumn]
location: Optional[str]
input_format: str
output_format: str
serde: str
properties: Dict[str, str]
comment: Optional[str]
create_time: datetime
last_access_time: datetime
owner: str
partitions: List[HivePartition]
@dataclass
class HiveQuery:
"""Hive查询"""
query_id: str
sql: str
database: str
status: HiveQueryStatus
start_time: datetime
end_time: Optional[datetime]
user: str
queue: str
application_id: Optional[str]
progress: float
error_message: Optional[str]
result_location: Optional[str]
rows_affected: int
bytes_read: int
bytes_written: int
class HiveMetastore:
"""
Hive元数据存储管理器
"""
def __init__(self):
self.databases = {}
self.tables = {}
self.queries = {}
self.query_counter = 0
# 创建默认数据库
self.create_database("default", "Default Hive database")
def create_database(self, db_name: str, comment: Optional[str] = None,
location: Optional[str] = None) -> Dict[str, Any]:
"""
创建数据库
Args:
db_name: 数据库名称
comment: 注释
location: 存储位置
Returns:
Dict[str, Any]: 创建结果
"""
if db_name in self.databases:
return {'status': 'error', 'message': f'Database {db_name} already exists'}
database = {
'name': db_name,
'comment': comment,
'location': location or f'/user/hive/warehouse/{db_name}.db',
'owner': 'hive',
'create_time': datetime.now(),
'tables': []
}
self.databases[db_name] = database
return {
'status': 'success',
'database': db_name,
'location': database['location'],
'create_time': database['create_time'].isoformat()
}
def drop_database(self, db_name: str, cascade: bool = False) -> Dict[str, Any]:
"""
删除数据库
Args:
db_name: 数据库名称
cascade: 是否级联删除表
Returns:
Dict[str, Any]: 删除结果
"""
if db_name not in self.databases:
return {'status': 'error', 'message': f'Database {db_name} does not exist'}
if db_name == 'default':
return {'status': 'error', 'message': 'Cannot drop default database'}
database = self.databases[db_name]
# 检查是否有表
if database['tables'] and not cascade:
return {
'status': 'error',
'message': f'Database {db_name} is not empty. Use CASCADE to drop.'
}
# 删除数据库中的所有表
if cascade:
for table_name in database['tables'].copy():
self.drop_table(db_name, table_name)
del self.databases[db_name]
return {
'status': 'success',
'database': db_name,
'dropped_tables': len(database['tables']) if cascade else 0
}
def create_table(self, table_config: Dict[str, Any]) -> Dict[str, Any]:
"""
创建表
Args:
table_config: 表配置
Returns:
Dict[str, Any]: 创建结果
"""
db_name = table_config.get('database', 'default')
table_name = table_config['table_name']
if db_name not in self.databases:
return {'status': 'error', 'message': f'Database {db_name} does not exist'}
table_key = f"{db_name}.{table_name}"
if table_key in self.tables:
return {'status': 'error', 'message': f'Table {table_key} already exists'}
# 解析列定义
columns = []
for col_def in table_config.get('columns', []):
columns.append(HiveColumn(
name=col_def['name'],
data_type=HiveDataType(col_def['type']),
comment=col_def.get('comment'),
nullable=col_def.get('nullable', True)
))
# 解析分区列
partition_columns = []
for part_col in table_config.get('partition_columns', []):
partition_columns.append(HiveColumn(
name=part_col['name'],
data_type=HiveDataType(part_col['type']),
comment=part_col.get('comment')
))
# 创建表对象
table = HiveTable(
database=db_name,
table_name=table_name,
table_type=HiveTableType(table_config.get('table_type', 'MANAGED')),
columns=columns,
partition_columns=partition_columns,
location=table_config.get('location'),
input_format=table_config.get('input_format', 'org.apache.hadoop.mapred.TextInputFormat'),
output_format=table_config.get('output_format', 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'),
serde=table_config.get('serde', 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'),
properties=table_config.get('properties', {}),
comment=table_config.get('comment'),
create_time=datetime.now(),
last_access_time=datetime.now(),
owner=table_config.get('owner', 'hive'),
partitions=[]
)
self.tables[table_key] = table
self.databases[db_name]['tables'].append(table_name)
return {
'status': 'success',
'database': db_name,
'table': table_name,
'table_type': table.table_type.value,
'columns': len(columns),
'partition_columns': len(partition_columns),
'location': table.location,
'create_time': table.create_time.isoformat()
}
def drop_table(self, db_name: str, table_name: str) -> Dict[str, Any]:
"""
删除表
Args:
db_name: 数据库名称
table_name: 表名称
Returns:
Dict[str, Any]: 删除结果
"""
table_key = f"{db_name}.{table_name}"
if table_key not in self.tables:
return {'status': 'error', 'message': f'Table {table_key} does not exist'}
table = self.tables[table_key]
# 从数据库中移除表
if db_name in self.databases and table_name in self.databases[db_name]['tables']:
self.databases[db_name]['tables'].remove(table_name)
del self.tables[table_key]
return {
'status': 'success',
'database': db_name,
'table': table_name,
'table_type': table.table_type.value,
'partitions_dropped': len(table.partitions)
}
def add_partition(self, db_name: str, table_name: str,
partition_spec: Dict[str, str],
location: Optional[str] = None) -> Dict[str, Any]:
"""
添加分区
Args:
db_name: 数据库名称
table_name: 表名称
partition_spec: 分区规格
location: 分区位置
Returns:
Dict[str, Any]: 添加结果
"""
table_key = f"{db_name}.{table_name}"
if table_key not in self.tables:
return {'status': 'error', 'message': f'Table {table_key} does not exist'}
table = self.tables[table_key]
# 检查分区列是否匹配
partition_columns = {col.name for col in table.partition_columns}
if not partition_columns:
return {'status': 'error', 'message': f'Table {table_key} is not partitioned'}
if set(partition_spec.keys()) != partition_columns:
return {
'status': 'error',
'message': f'Partition spec {partition_spec} does not match partition columns {partition_columns}'
}
# 检查分区是否已存在
for existing_partition in table.partitions:
if all(existing_partition.column == col and existing_partition.value == val
for col, val in partition_spec.items()):
return {'status': 'error', 'message': f'Partition {partition_spec} already exists'}
# 添加分区
for col, val in partition_spec.items():
partition = HivePartition(
column=col,
value=val,
location=location or f"{table.location}/{col}={val}"
)
table.partitions.append(partition)
return {
'status': 'success',
'database': db_name,
'table': table_name,
'partition': partition_spec,
'location': location or f"{table.location}/{list(partition_spec.items())[0][0]}={list(partition_spec.items())[0][1]}"
}
def drop_partition(self, db_name: str, table_name: str,
partition_spec: Dict[str, str]) -> Dict[str, Any]:
"""
删除分区
Args:
db_name: 数据库名称
table_name: 表名称
partition_spec: 分区规格
Returns:
Dict[str, Any]: 删除结果
"""
table_key = f"{db_name}.{table_name}"
if table_key not in self.tables:
return {'status': 'error', 'message': f'Table {table_key} does not exist'}
table = self.tables[table_key]
# 查找并删除分区
partitions_to_remove = []
for partition in table.partitions:
if partition.column in partition_spec and partition.value == partition_spec[partition.column]:
partitions_to_remove.append(partition)
if not partitions_to_remove:
return {'status': 'error', 'message': f'Partition {partition_spec} does not exist'}
for partition in partitions_to_remove:
table.partitions.remove(partition)
return {
'status': 'success',
'database': db_name,
'table': table_name,
'partition': partition_spec,
'partitions_dropped': len(partitions_to_remove)
}
def execute_query(self, sql: str, database: str = 'default',
user: str = 'hive', queue: str = 'default') -> Dict[str, Any]:
"""
执行查询
Args:
sql: SQL语句
database: 数据库
user: 用户
queue: 队列
Returns:
Dict[str, Any]: 查询结果
"""
self.query_counter += 1
query_id = f"query-{self.query_counter:06d}"
# 创建查询对象
query = HiveQuery(
query_id=query_id,
sql=sql.strip(),
database=database,
status=HiveQueryStatus.PENDING,
start_time=datetime.now(),
end_time=None,
user=user,
queue=queue,
application_id=None,
progress=0.0,
error_message=None,
result_location=None,
rows_affected=0,
bytes_read=0,
bytes_written=0
)
self.queries[query_id] = query
# 解析并执行SQL
try:
result = self._parse_and_execute_sql(query)
query.status = HiveQueryStatus.FINISHED
query.end_time = datetime.now()
query.progress = 100.0
return {
'status': 'success',
'query_id': query_id,
'result': result,
'execution_time_ms': int((query.end_time - query.start_time).total_seconds() * 1000),
'rows_affected': query.rows_affected
}
except Exception as e:
query.status = HiveQueryStatus.ERROR
query.end_time = datetime.now()
query.error_message = str(e)
return {
'status': 'error',
'query_id': query_id,
'error': str(e),
'execution_time_ms': int((query.end_time - query.start_time).total_seconds() * 1000)
}
def _parse_and_execute_sql(self, query: HiveQuery) -> Dict[str, Any]:
"""
解析并执行SQL语句
Args:
query: 查询对象
Returns:
Dict[str, Any]: 执行结果
"""
sql = query.sql.upper().strip()
if sql.startswith('CREATE DATABASE'):
return self._execute_create_database(query)
elif sql.startswith('DROP DATABASE'):
return self._execute_drop_database(query)
elif sql.startswith('CREATE TABLE'):
return self._execute_create_table(query)
elif sql.startswith('DROP TABLE'):
return self._execute_drop_table(query)
elif sql.startswith('SHOW DATABASES'):
return self._execute_show_databases(query)
elif sql.startswith('SHOW TABLES'):
return self._execute_show_tables(query)
elif sql.startswith('DESCRIBE') or sql.startswith('DESC'):
return self._execute_describe_table(query)
elif sql.startswith('SELECT'):
return self._execute_select(query)
elif sql.startswith('INSERT'):
return self._execute_insert(query)
else:
raise ValueError(f"Unsupported SQL statement: {sql[:50]}...")
def _execute_show_databases(self, query: HiveQuery) -> Dict[str, Any]:
"""
执行SHOW DATABASES
Args:
query: 查询对象
Returns:
Dict[str, Any]: 执行结果
"""
databases = list(self.databases.keys())
query.rows_affected = len(databases)
return {
'columns': ['database_name'],
'data': [[db] for db in databases],
'row_count': len(databases)
}
def _execute_show_tables(self, query: HiveQuery) -> Dict[str, Any]:
"""
执行SHOW TABLES
Args:
query: 查询对象
Returns:
Dict[str, Any]: 执行结果
"""
db_name = query.database
if db_name not in self.databases:
raise ValueError(f"Database {db_name} does not exist")
tables = self.databases[db_name]['tables']
query.rows_affected = len(tables)
return {
'columns': ['tab_name'],
'data': [[table] for table in tables],
'row_count': len(tables)
}
def _execute_describe_table(self, query: HiveQuery) -> Dict[str, Any]:
"""
执行DESCRIBE TABLE
Args:
query: 查询对象
Returns:
Dict[str, Any]: 执行结果
"""
# 简单解析表名
sql_parts = query.sql.split()
if len(sql_parts) < 2:
raise ValueError("Invalid DESCRIBE statement")
table_name = sql_parts[1]
if '.' in table_name:
db_name, table_name = table_name.split('.', 1)
else:
db_name = query.database
table_key = f"{db_name}.{table_name}"
if table_key not in self.tables:
raise ValueError(f"Table {table_key} does not exist")
table = self.tables[table_key]
# 构建结果
data = []
for col in table.columns:
data.append([col.name, col.data_type.value, col.comment or ''])
# 添加分区列
if table.partition_columns:
data.append(['', '', ''])
data.append(['# Partition Information', '', ''])
data.append(['# col_name', 'data_type', 'comment'])
for col in table.partition_columns:
data.append([col.name, col.data_type.value, col.comment or ''])
query.rows_affected = len(data)
return {
'columns': ['col_name', 'data_type', 'comment'],
'data': data,
'row_count': len(data)
}
def _execute_select(self, query: HiveQuery) -> Dict[str, Any]:
"""
执行SELECT语句（模拟）
Args:
query: 查询对象
Returns:
Dict[str, Any]: 执行结果
"""
# 简单模拟SELECT结果
query.rows_affected = 100 # 模拟返回100行
query.bytes_read = 1024 * 1024 # 模拟读取1MB数据
return {
'columns': ['col1', 'col2', 'col3'],
'data': [['value1', 'value2', 'value3'] for _ in range(10)], # 只返回前10行作为示例
'row_count': 100,
'note': 'This is a simulated result. In real Hive, data would be read from HDFS.'
}
def _execute_insert(self, query: HiveQuery) -> Dict[str, Any]:
"""
执行INSERT语句（模拟）
Args:
query: 查询对象
Returns:
Dict[str, Any]: 执行结果
"""
# 简单模拟INSERT结果
query.rows_affected = 1000 # 模拟插入1000行
query.bytes_written = 5 * 1024 * 1024 # 模拟写入5MB数据
return {
'message': 'Insert operation completed',
'rows_inserted': 1000,
'note': 'This is a simulated result. In real Hive, data would be written to HDFS.'
}
def get_table_info(self, db_name: str, table_name: str) -> Dict[str, Any]:
"""
获取表信息
Args:
db_name: 数据库名称
table_name: 表名称
Returns:
Dict[str, Any]: 表信息
"""
table_key = f"{db_name}.{table_name}"
if table_key not in self.tables:
return {'status': 'error', 'message': f'Table {table_key} does not exist'}
table = self.tables[table_key]
return {
'database': table.database,
'table_name': table.table_name,
'table_type': table.table_type.value,
'owner': table.owner,
'create_time': table.create_time.isoformat(),
'last_access_time': table.last_access_time.isoformat(),
'location': table.location,
'input_format': table.input_format,
'output_format': table.output_format,
'serde': table.serde,
'comment': table.comment,
'columns': [
{
'name': col.name,
'type': col.data_type.value,
'comment': col.comment,
'nullable': col.nullable
} for col in table.columns
],
'partition_columns': [
{
'name': col.name,
'type': col.data_type.value,
'comment': col.comment
} for col in table.partition_columns
],
'partitions': [
{
'column': part.column,
'value': part.value,
'location': part.location
} for part in table.partitions
],
'properties': table.properties
}
def get_query_status(self, query_id: str) -> Dict[str, Any]:
"""
获取查询状态
Args:
query_id: 查询ID
Returns:
Dict[str, Any]: 查询状态
"""
if query_id not in self.queries:
return {'status': 'error', 'message': f'Query {query_id} not found'}
query = self.queries[query_id]
result = {
'query_id': query.query_id,
'sql': query.sql,
'database': query.database,
'status': query.status.value,
'user': query.user,
'queue': query.queue,
'start_time': query.start_time.isoformat(),
'progress': query.progress,
'rows_affected': query.rows_affected,
'bytes_read': query.bytes_read,
'bytes_written': query.bytes_written
}
if query.end_time:
result['end_time'] = query.end_time.isoformat()
result['duration_ms'] = int((query.end_time - query.start_time).total_seconds() * 1000)
if query.error_message:
result['error_message'] = query.error_message
if query.application_id:
result['application_id'] = query.application_id
if query.result_location:
result['result_location'] = query.result_location
return result
def get_database_info(self, db_name: str) -> Dict[str, Any]:
"""
获取数据库信息
Args:
db_name: 数据库名称
Returns:
Dict[str, Any]: 数据库信息
"""
if db_name not in self.databases:
return {'status': 'error', 'message': f'Database {db_name} does not exist'}
database = self.databases[db_name]
return {
'name': database['name'],
'comment': database['comment'],
'location': database['location'],
'owner': database['owner'],
'create_time': database['create_time'].isoformat(),
'table_count': len(database['tables']),
'tables': database['tables']
}
def get_cluster_stats(self) -> Dict[str, Any]:
"""
获取集群统计信息
Returns:
Dict[str, Any]: 集群统计
"""
total_tables = len(self.tables)
total_partitions = sum(len(table.partitions) for table in self.tables.values())
# 按类型统计表
table_types = {}
for table in self.tables.values():
table_type = table.table_type.value
table_types[table_type] = table_types.get(table_type, 0) + 1
# 统计查询
query_stats = {}
for query in self.queries.values():
status = query.status.value
query_stats[status] = query_stats.get(status, 0) + 1
return {
'databases': {
'total': len(self.databases),
'names': list(self.databases.keys())
},
'tables': {
'total': total_tables,
'by_type': table_types,
'total_partitions': total_partitions
},
'queries': {
'total': len(self.queries),
'by_status': query_stats
},
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Hive元数据存储
hive = HiveMetastore()
print("=== Hive元数据管理示例 ===")
# 创建数据库
print("\n=== 创建数据库 ===")
db_result = hive.create_database('sales', 'Sales data warehouse')
print(f"数据库创建结果: {db_result}")
# 创建表
print("\n=== 创建表 ===")
table_config = {
'database': 'sales',
'table_name': 'orders',
'table_type': 'MANAGED',
'columns': [
{'name': 'order_id', 'type': 'bigint', 'comment': '订单ID'},
{'name': 'customer_id', 'type': 'bigint', 'comment': '客户ID'},
{'name': 'product_name', 'type': 'string', 'comment': '产品名称'},
{'name': 'quantity', 'type': 'int', 'comment': '数量'},
{'name': 'price', 'type': 'double', 'comment': '价格'},
{'name': 'order_date', 'type': 'date', 'comment': '订单日期'}
],
'partition_columns': [
{'name': 'year', 'type': 'int', 'comment': '年份'},
{'name': 'month', 'type': 'int', 'comment': '月份'}
],
'comment': '订单表',
'properties': {
'serialization.format': '1',
'field.delim': '\t'
}
}
table_result = hive.create_table(table_config)
print(f"表创建结果: {table_result}")
# 添加分区
print("\n=== 添加分区 ===")
partition_result = hive.add_partition('sales', 'orders', {'year': '2024', 'month': '1'})
print(f"分区添加结果: {partition_result}")
partition_result = hive.add_partition('sales', 'orders', {'year': '2024', 'month': '2'})
print(f"分区添加结果: {partition_result}")
# 执行查询
print("\n=== 执行查询 ===")
# SHOW DATABASES
query_result = hive.execute_query('SHOW DATABASES')
print(f"SHOW DATABASES结果: {query_result}")
# SHOW TABLES
query_result = hive.execute_query('SHOW TABLES', database='sales')
print(f"SHOW TABLES结果: {query_result}")
# DESCRIBE TABLE
query_result = hive.execute_query('DESCRIBE sales.orders')
print(f"DESCRIBE TABLE结果: {query_result}")
# SELECT查询
query_result = hive.execute_query(
'SELECT order_id, customer_id, product_name FROM sales.orders WHERE year=2024 AND month=1',
database='sales'
)
print(f"SELECT查询结果: {query_result}")
# 获取表信息
print("\n=== 表信息 ===")
table_info = hive.get_table_info('sales', 'orders')
print(f"表名: {table_info['table_name']}")
print(f"表类型: {table_info['table_type']}")
print(f"列数: {len(table_info['columns'])}")
print(f"分区列数: {len(table_info['partition_columns'])}")
print(f"分区数: {len(table_info['partitions'])}")
# 获取集群统计
print("\n=== 集群统计 ===")
stats = hive.get_cluster_stats()
print(f"数据库总数: {stats['databases']['total']}")
print(f"表总数: {stats['tables']['total']}")
print(f"分区总数: {stats['tables']['total_partitions']}")
print(f"查询总数: {stats['queries']['total']}")
print("\n数据库列表:")
for db_name in stats['databases']['names']:
db_info = hive.get_database_info(db_name)
print(f" {db_name}: {db_info['table_count']} tables")
4.2 Apache Impala详解
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass
from enum import Enum
from datetime import datetime
import threading
import time
import random
class ImpalaQueryState(Enum):
"""Impala查询状态"""
CREATED = "已创建"
INITIALIZED = "已初始化"
COMPILED = "已编译"
RUNNING = "运行中"
FINISHED = "已完成"
CANCELLED = "已取消"
EXCEPTION = "异常"
class ImpalaQueryType(Enum):
"""Impala查询类型"""
QUERY = "查询"
DML = "数据操作"
DDL = "数据定义"
UTILITY = "工具"
class ImpalaResourcePool(Enum):
"""Impala资源池"""
DEFAULT = "default-pool"
SMALL = "small-pool"
LARGE = "large-pool"
BATCH = "batch-pool"
@dataclass
class ImpalaQueryProfile:
"""Impala查询配置文件"""
query_id: str
sql_statement: str
query_type: ImpalaQueryType
start_time: datetime
end_time: Optional[datetime]
state: ImpalaQueryState
user: str
database: str
resource_pool: ImpalaResourcePool
coordinator: str
progress: float
rows_produced: int
peak_memory_usage: int
spilled_memory: int
hdfs_bytes_read: int
hdfs_bytes_written: int
scan_ranges: int
cpu_time_ms: int
network_address: str
error_message: Optional[str]
warnings: List[str]
plan_root_id: Optional[str]
estimated_per_host_mem: int
tables_missing_stats: List[str]
admission_result: str
query_options: Dict[str, Any]
@dataclass
class ImpalaNode:
"""Impala节点"""
hostname: str
port: int
is_coordinator: bool
is_executor: bool
version: str
memory_limit: int
memory_reserved: int
memory_used: int
num_queries: int
num_fragments: int
cpu_usage: float
network_throughput: float
disk_io_mgr_queue_size: int
thread_manager_queue_size: int
status: str
last_heartbeat: datetime
@dataclass
class ImpalaQueryExecution:
"""Impala查询执行信息"""
fragment_id: str
instance_id: str
host: str
state: str
start_time: datetime
end_time: Optional[datetime]
rows_produced: int
memory_used: int
cpu_time: int
scan_ranges_complete: int
scan_ranges_total: int
hdfs_read_timer: int
per_host_peak_mem_usage: int
class ImpalaCluster:
"""
Impala集群管理器
"""
def __init__(self, cluster_name: str = "impala-cluster"):
self.cluster_name = cluster_name
self.nodes = {}
self.queries = {}
self.query_counter = 0
self.resource_pools = {
ImpalaResourcePool.DEFAULT: {
'max_memory': 8 * 1024 * 1024 * 1024, # 8GB
'max_queries': 200,
'max_queued': 50,
'queue_timeout_ms': 60000,
'default_query_options': {}
},
ImpalaResourcePool.SMALL: {
'max_memory': 2 * 1024 * 1024 * 1024, # 2GB
'max_queries': 50,
'max_queued': 20,
'queue_timeout_ms': 30000,
'default_query_options': {'mem_limit': '1GB'}
},
ImpalaResourcePool.LARGE: {
'max_memory': 32 * 1024 * 1024 * 1024, # 32GB
'max_queries': 10,
'max_queued': 5,
'queue_timeout_ms': 300000,
'default_query_options': {'mem_limit': '16GB'}
},
ImpalaResourcePool.BATCH: {
'max_memory': 64 * 1024 * 1024 * 1024, # 64GB
'max_queries': 5,
'max_queued': 10,
'queue_timeout_ms': 600000,
'default_query_options': {'mem_limit': '32GB', 'batch_size': '1024'}
}
}
self.catalog_version = 1
self.admission_controller_stats = {
'total_admitted': 0,
'total_queued': 0,
'total_rejected': 0,
'total_timed_out': 0
}
# 初始化默认节点
self._initialize_default_nodes()
def _initialize_default_nodes(self):
"""
初始化默认节点
"""
# 添加协调器节点
self.add_node(
hostname="impala-coordinator-1",
port=21000,
is_coordinator=True,
is_executor=True,
memory_limit=16 * 1024 * 1024 * 1024 # 16GB
)
# 添加执行器节点
for i in range(2, 5):
self.add_node(
hostname=f"impala-executor-{i}",
port=22000,
is_coordinator=False,
is_executor=True,
memory_limit=32 * 1024 * 1024 * 1024 # 32GB
)
def add_node(self, hostname: str, port: int, is_coordinator: bool = False,
is_executor: bool = True, memory_limit: int = 16 * 1024 * 1024 * 1024,
version: str = "4.0.0") -> Dict[str, Any]:
"""
添加Impala节点
Args:
hostname: 主机名
port: 端口
is_coordinator: 是否为协调器
is_executor: 是否为执行器
memory_limit: 内存限制
version: 版本
Returns:
Dict[str, Any]: 添加结果
"""
node_id = f"{hostname}:{port}"
if node_id in self.nodes:
return {'status': 'error', 'message': f'Node {node_id} already exists'}
node = ImpalaNode(
hostname=hostname,
port=port,
is_coordinator=is_coordinator,
is_executor=is_executor,
version=version,
memory_limit=memory_limit,
memory_reserved=0,
memory_used=random.randint(1024 * 1024 * 1024, memory_limit // 4), # 随机使用内存
num_queries=0,
num_fragments=0,
cpu_usage=random.uniform(10.0, 80.0), # 随机CPU使用率
network_throughput=random.uniform(100.0, 1000.0), # 随机网络吞吐量
disk_io_mgr_queue_size=random.randint(0, 10),
thread_manager_queue_size=random.randint(0, 5),
status="healthy",
last_heartbeat=datetime.now()
)
self.nodes[node_id] = node
return {
'status': 'success',
'node_id': node_id,
'hostname': hostname,
'port': port,
'is_coordinator': is_coordinator,
'is_executor': is_executor,
'memory_limit_gb': memory_limit // (1024 * 1024 * 1024)
}
def remove_node(self, hostname: str, port: int) -> Dict[str, Any]:
"""
移除Impala节点
Args:
hostname: 主机名
port: 端口
Returns:
Dict[str, Any]: 移除结果
"""
node_id = f"{hostname}:{port}"
if node_id not in self.nodes:
return {'status': 'error', 'message': f'Node {node_id} does not exist'}
node = self.nodes[node_id]
# 检查节点是否有运行中的查询
running_queries = [q for q in self.queries.values()
if q.state == ImpalaQueryState.RUNNING and q.coordinator == node_id]
if running_queries:
return {
'status': 'error',
'message': f'Node {node_id} has {len(running_queries)} running queries'
}
del self.nodes[node_id]
return {
'status': 'success',
'node_id': node_id,
'running_queries': node.num_queries,
'memory_used_gb': node.memory_used // (1024 * 1024 * 1024)
}
def submit_query(self, sql: str, user: str = "impala", database: str = "default",
resource_pool: ImpalaResourcePool = ImpalaResourcePool.DEFAULT,
query_options: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
提交查询
Args:
sql: SQL语句
user: 用户
database: 数据库
resource_pool: 资源池
query_options: 查询选项
Returns:
Dict[str, Any]: 提交结果
"""
self.query_counter += 1
query_id = f"impala-query-{self.query_counter:08d}"
# 选择协调器
coordinators = [node_id for node_id, node in self.nodes.items() if node.is_coordinator]
if not coordinators:
return {'status': 'error', 'message': 'No coordinator nodes available'}
coordinator = random.choice(coordinators)
# 检查资源池准入
admission_result = self._check_admission_control(resource_pool)
if admission_result['status'] != 'admitted':
self.admission_controller_stats['total_rejected'] += 1
return {
'status': 'error',
'message': f'Query rejected by admission controller: {admission_result["reason"]}',
'query_id': query_id
}
# 确定查询类型
query_type = self._determine_query_type(sql)
# 合并查询选项
pool_options = self.resource_pools[resource_pool]['default_query_options'].copy()
if query_options:
pool_options.update(query_options)
# 创建查询配置文件
query_profile = ImpalaQueryProfile(
query_id=query_id,
sql_statement=sql.strip(),
query_type=query_type,
start_time=datetime.now(),
end_time=None,
state=ImpalaQueryState.CREATED,
user=user,
database=database,
resource_pool=resource_pool,
coordinator=coordinator,
progress=0.0,
rows_produced=0,
peak_memory_usage=0,
spilled_memory=0,
hdfs_bytes_read=0,
hdfs_bytes_written=0,
scan_ranges=0,
cpu_time_ms=0,
network_address=coordinator,
error_message=None,
warnings=[],
plan_root_id=None,
estimated_per_host_mem=pool_options.get('mem_limit', 2 * 1024 * 1024 * 1024),
tables_missing_stats=[],
admission_result=admission_result['status'],
query_options=pool_options
)
self.queries[query_id] = query_profile
self.admission_controller_stats['total_admitted'] += 1
# 更新协调器统计
self.nodes[coordinator].num_queries += 1
return {
'status': 'success',
'query_id': query_id,
'coordinator': coordinator,
'resource_pool': resource_pool.value,
'estimated_memory': query_profile.estimated_per_host_mem,
'admission_result': admission_result['status']
}
def _check_admission_control(self, resource_pool: ImpalaResourcePool) -> Dict[str, Any]:
"""
检查准入控制
Args:
resource_pool: 资源池
Returns:
Dict[str, Any]: 准入结果
"""
pool_config = self.resource_pools[resource_pool]
# 统计当前资源池中的查询
running_queries = [q for q in self.queries.values()
if q.resource_pool == resource_pool and q.state == ImpalaQueryState.RUNNING]
queued_queries = [q for q in self.queries.values()
if q.resource_pool == resource_pool and q.state == ImpalaQueryState.CREATED]
# 检查查询数量限制
if len(running_queries) >= pool_config['max_queries']:
if len(queued_queries) >= pool_config['max_queued']:
return {
'status': 'rejected',
'reason': f'Resource pool {resource_pool.value} queue is full'
}
else:
return {
'status': 'queued',
'reason': f'Resource pool {resource_pool.value} is at capacity, query queued'
}
# 检查内存限制
total_memory_used = sum(q.peak_memory_usage for q in running_queries)
if total_memory_used >= pool_config['max_memory']:
return {
'status': 'rejected',
'reason': f'Resource pool {resource_pool.value} memory limit exceeded'
}
return {'status': 'admitted', 'reason': 'Query admitted to resource pool'}
def _determine_query_type(self, sql: str) -> ImpalaQueryType:
"""
确定查询类型
Args:
sql: SQL语句
Returns:
ImpalaQueryType: 查询类型
"""
sql_upper = sql.upper().strip()
if sql_upper.startswith(('SELECT', 'WITH')):
return ImpalaQueryType.QUERY
elif sql_upper.startswith(('INSERT', 'UPDATE', 'DELETE', 'UPSERT')):
return ImpalaQueryType.DML
elif sql_upper.startswith(('CREATE', 'DROP', 'ALTER', 'TRUNCATE')):
return ImpalaQueryType.DDL
else:
return ImpalaQueryType.UTILITY
def execute_query(self, query_id: str) -> Dict[str, Any]:
"""
执行查询
Args:
query_id: 查询ID
Returns:
Dict[str, Any]: 执行结果
"""
if query_id not in self.queries:
return {'status': 'error', 'message': f'Query {query_id} not found'}
query = self.queries[query_id]
if query.state != ImpalaQueryState.CREATED:
return {
'status': 'error',
'message': f'Query {query_id} is in state {query.state.value}, cannot execute'
}
try:
# 模拟查询执行过程
self._simulate_query_execution(query)
return {
'status': 'success',
'query_id': query_id,
'state': query.state.value,
'rows_produced': query.rows_produced,
'execution_time_ms': int((query.end_time - query.start_time).total_seconds() * 1000) if query.end_time else 0,
'peak_memory_usage': query.peak_memory_usage,
'hdfs_bytes_read': query.hdfs_bytes_read
}
except Exception as e:
query.state = ImpalaQueryState.EXCEPTION
query.error_message = str(e)
query.end_time = datetime.now()
return {
'status': 'error',
'query_id': query_id,
'error': str(e),
'state': query.state.value
}
def _simulate_query_execution(self, query: ImpalaQueryProfile):
"""
模拟查询执行
Args:
query: 查询配置文件
"""
# 初始化阶段
query.state = ImpalaQueryState.INITIALIZED
time.sleep(0.1) # 模拟初始化时间
# 编译阶段
query.state = ImpalaQueryState.COMPILED
query.plan_root_id = f"plan-{random.randint(1000, 9999)}"
time.sleep(0.2) # 模拟编译时间
# 运行阶段
query.state = ImpalaQueryState.RUNNING
# 模拟执行统计
execution_time = random.uniform(0.5, 5.0) # 0.5-5秒执行时间
query.rows_produced = random.randint(100, 10000)
query.peak_memory_usage = random.randint(100 * 1024 * 1024, 2 * 1024 * 1024 * 1024) # 100MB-2GB
query.hdfs_bytes_read = random.randint(1024 * 1024, 100 * 1024 * 1024) # 1MB-100MB
query.scan_ranges = random.randint(1, 50)
query.cpu_time_ms = int(execution_time * 1000 * random.uniform(0.3, 0.8))
# 模拟执行时间
time.sleep(execution_time)
# 完成阶段
query.state = ImpalaQueryState.FINISHED
query.end_time = datetime.now()
query.progress = 100.0
# 更新节点统计
coordinator_node = self.nodes[query.coordinator]
coordinator_node.memory_used += query.peak_memory_usage
coordinator_node.num_queries -= 1
def cancel_query(self, query_id: str) -> Dict[str, Any]:
"""
取消查询
Args:
query_id: 查询ID
Returns:
Dict[str, Any]: 取消结果
"""
if query_id not in self.queries:
return {'status': 'error', 'message': f'Query {query_id} not found'}
query = self.queries[query_id]
if query.state in [ImpalaQueryState.FINISHED, ImpalaQueryState.CANCELLED, ImpalaQueryState.EXCEPTION]:
return {
'status': 'error',
'message': f'Query {query_id} is already in terminal state {query.state.value}'
}
query.state = ImpalaQueryState.CANCELLED
query.end_time = datetime.now()
# 更新节点统计
if query.coordinator in self.nodes:
self.nodes[query.coordinator].num_queries -= 1
return {
'status': 'success',
'query_id': query_id,
'state': query.state.value,
'cancelled_at': query.end_time.isoformat()
}
def get_query_profile(self, query_id: str) -> Dict[str, Any]:
"""
获取查询配置文件
Args:
query_id: 查询ID
Returns:
Dict[str, Any]: 查询配置文件
"""
if query_id not in self.queries:
return {'status': 'error', 'message': f'Query {query_id} not found'}
query = self.queries[query_id]
profile = {
'query_id': query.query_id,
'sql_statement': query.sql_statement,
'query_type': query.query_type.value,
'state': query.state.value,
'user': query.user,
'database': query.database,
'resource_pool': query.resource_pool.value,
'coordinator': query.coordinator,
'start_time': query.start_time.isoformat(),
'progress': query.progress,
'rows_produced': query.rows_produced,
'peak_memory_usage': query.peak_memory_usage,
'spilled_memory': query.spilled_memory,
'hdfs_bytes_read': query.hdfs_bytes_read,
'hdfs_bytes_written': query.hdfs_bytes_written,
'scan_ranges': query.scan_ranges,
'cpu_time_ms': query.cpu_time_ms,
'network_address': query.network_address,
'plan_root_id': query.plan_root_id,
'estimated_per_host_mem': query.estimated_per_host_mem,
'tables_missing_stats': query.tables_missing_stats,
'admission_result': query.admission_result,
'query_options': query.query_options,
'warnings': query.warnings
}
if query.end_time:
profile['end_time'] = query.end_time.isoformat()
profile['duration_ms'] = int((query.end_time - query.start_time).total_seconds() * 1000)
if query.error_message:
profile['error_message'] = query.error_message
return profile
def get_cluster_status(self) -> Dict[str, Any]:
"""
获取集群状态
Returns:
Dict[str, Any]: 集群状态
"""
# 统计节点
total_nodes = len(self.nodes)
healthy_nodes = sum(1 for node in self.nodes.values() if node.status == "healthy")
coordinators = sum(1 for node in self.nodes.values() if node.is_coordinator)
executors = sum(1 for node in self.nodes.values() if node.is_executor)
# 统计内存
total_memory = sum(node.memory_limit for node in self.nodes.values())
used_memory = sum(node.memory_used for node in self.nodes.values())
reserved_memory = sum(node.memory_reserved for node in self.nodes.values())
# 统计查询
query_stats = {}
for query in self.queries.values():
state = query.state.value
query_stats[state] = query_stats.get(state, 0) + 1
# 统计资源池
pool_stats = {}
for pool in ImpalaResourcePool:
pool_queries = [q for q in self.queries.values() if q.resource_pool == pool]
pool_stats[pool.value] = {
'total_queries': len(pool_queries),
'running_queries': len([q for q in pool_queries if q.state == ImpalaQueryState.RUNNING]),
'queued_queries': len([q for q in pool_queries if q.state == ImpalaQueryState.CREATED])
}
return {
'cluster_name': self.cluster_name,
'catalog_version': self.catalog_version,
'nodes': {
'total': total_nodes,
'healthy': healthy_nodes,
'coordinators': coordinators,
'executors': executors
},
'memory': {
'total_gb': total_memory // (1024 * 1024 * 1024),
'used_gb': used_memory // (1024 * 1024 * 1024),
'reserved_gb': reserved_memory // (1024 * 1024 * 1024),
'utilization_percent': (used_memory / total_memory * 100) if total_memory > 0 else 0
},
'queries': {
'total': len(self.queries),
'by_state': query_stats
},
'resource_pools': pool_stats,
'admission_controller': self.admission_controller_stats,
'timestamp': datetime.now().isoformat()
}
def get_node_status(self, hostname: str, port: int) -> Dict[str, Any]:
"""
获取节点状态
Args:
hostname: 主机名
port: 端口
Returns:
Dict[str, Any]: 节点状态
"""
node_id = f"{hostname}:{port}"
if node_id not in self.nodes:
return {'status': 'error', 'message': f'Node {node_id} not found'}
node = self.nodes[node_id]
# 统计该节点的查询
node_queries = [q for q in self.queries.values() if q.coordinator == node_id]
return {
'node_id': node_id,
'hostname': node.hostname,
'port': node.port,
'is_coordinator': node.is_coordinator,
'is_executor': node.is_executor,
'version': node.version,
'status': node.status,
'last_heartbeat': node.last_heartbeat.isoformat(),
'memory': {
'limit_gb': node.memory_limit // (1024 * 1024 * 1024),
'reserved_gb': node.memory_reserved // (1024 * 1024 * 1024),
'used_gb': node.memory_used // (1024 * 1024 * 1024),
'utilization_percent': (node.memory_used / node.memory_limit * 100) if node.memory_limit > 0 else 0
},
'performance': {
'cpu_usage_percent': node.cpu_usage,
'network_throughput_mbps': node.network_throughput,
'disk_io_mgr_queue_size': node.disk_io_mgr_queue_size,
'thread_manager_queue_size': node.thread_manager_queue_size
},
'queries': {
'current': node.num_queries,
'total_handled': len(node_queries),
'fragments': node.num_fragments
}
}
def get_resource_pool_stats(self, pool: ImpalaResourcePool) -> Dict[str, Any]:
"""
获取资源池统计
Args:
pool: 资源池
Returns:
Dict[str, Any]: 资源池统计
"""
pool_config = self.resource_pools[pool]
pool_queries = [q for q in self.queries.values() if q.resource_pool == pool]
# 按状态统计查询
state_stats = {}
for query in pool_queries:
state = query.state.value
state_stats[state] = state_stats.get(state, 0) + 1
# 计算内存使用
running_queries = [q for q in pool_queries if q.state == ImpalaQueryState.RUNNING]
total_memory_used = sum(q.peak_memory_usage for q in running_queries)
# 计算平均执行时间
finished_queries = [q for q in pool_queries if q.state == ImpalaQueryState.FINISHED and q.end_time]
avg_execution_time = 0
if finished_queries:
total_time = sum((q.end_time - q.start_time).total_seconds() for q in finished_queries)
avg_execution_time = total_time / len(finished_queries)
return {
'pool_name': pool.value,
'configuration': {
'max_memory_gb': pool_config['max_memory'] // (1024 * 1024 * 1024),
'max_queries': pool_config['max_queries'],
'max_queued': pool_config['max_queued'],
'queue_timeout_ms': pool_config['queue_timeout_ms']
},
'current_usage': {
'memory_used_gb': total_memory_used // (1024 * 1024 * 1024),
'memory_utilization_percent': (total_memory_used / pool_config['max_memory'] * 100) if pool_config['max_memory'] > 0 else 0,
'running_queries': len(running_queries),
'queued_queries': len([q for q in pool_queries if q.state == ImpalaQueryState.CREATED]),
'query_utilization_percent': (len(running_queries) / pool_config['max_queries'] * 100) if pool_config['max_queries'] > 0 else 0
},
'statistics': {
'total_queries': len(pool_queries),
'by_state': state_stats,
'avg_execution_time_seconds': avg_execution_time
},
'default_query_options': pool_config['default_query_options']
}
# 使用示例
if __name__ == "__main__":
# 创建Impala集群
impala = ImpalaCluster("production-impala")
print("=== Impala集群管理示例 ===")
# 添加额外节点
print("\n=== 添加节点 ===")
node_result = impala.add_node(
hostname="impala-executor-5",
port=22000,
is_coordinator=False,
is_executor=True,
memory_limit=64 * 1024 * 1024 * 1024 # 64GB
)
print(f"节点添加结果: {node_result}")
# 提交查询
print("\n=== 提交查询 ===")
# 提交SELECT查询
query_result = impala.submit_query(
sql="SELECT customer_id, SUM(amount) FROM sales WHERE year = 2024 GROUP BY customer_id",
user="analyst",
database="warehouse",
resource_pool=ImpalaResourcePool.DEFAULT
)
print(f"查询提交结果: {query_result}")
query_id_1 = query_result.get('query_id')
# 提交大查询
query_result = impala.submit_query(
sql="SELECT * FROM large_table JOIN another_large_table ON id = foreign_id",
user="data_scientist",
database="analytics",
resource_pool=ImpalaResourcePool.LARGE,
query_options={'mem_limit': '8GB', 'num_nodes': '4'}
)
print(f"大查询提交结果: {query_result}")
query_id_2 = query_result.get('query_id')
# 执行查询
print("\n=== 执行查询 ===")
if query_id_1:
exec_result = impala.execute_query(query_id_1)
print(f"查询执行结果: {exec_result}")
if query_id_2:
exec_result = impala.execute_query(query_id_2)
print(f"大查询执行结果: {exec_result}")
# 获取查询配置文件
print("\n=== 查询配置文件 ===")
if query_id_1:
profile = impala.get_query_profile(query_id_1)
print(f"查询ID: {profile['query_id']}")
print(f"状态: {profile['state']}")
print(f"执行时间: {profile.get('duration_ms', 0)}ms")
print(f"产生行数: {profile['rows_produced']}")
print(f"峰值内存: {profile['peak_memory_usage'] // (1024*1024)}MB")
print(f"HDFS读取: {profile['hdfs_bytes_read'] // (1024*1024)}MB")
# 获取集群状态
print("\n=== 集群状态 ===")
cluster_status = impala.get_cluster_status()
print(f"集群名称: {cluster_status['cluster_name']}")
print(f"节点总数: {cluster_status['nodes']['total']}")
print(f"健康节点: {cluster_status['nodes']['healthy']}")
print(f"协调器数: {cluster_status['nodes']['coordinators']}")
print(f"执行器数: {cluster_status['nodes']['executors']}")
print(f"内存使用率: {cluster_status['memory']['utilization_percent']:.1f}%")
print(f"查询总数: {cluster_status['queries']['total']}")
# 获取资源池统计
print("\n=== 资源池统计 ===")
for pool in ImpalaResourcePool:
pool_stats = impala.get_resource_pool_stats(pool)
print(f"\n{pool_stats['pool_name']}:")
print(f" 最大内存: {pool_stats['configuration']['max_memory_gb']}GB")
print(f" 最大查询数: {pool_stats['configuration']['max_queries']}")
print(f" 当前运行查询: {pool_stats['current_usage']['running_queries']}")
print(f" 内存使用率: {pool_stats['current_usage']['memory_utilization_percent']:.1f}%")
print(f" 查询使用率: {pool_stats['current_usage']['query_utilization_percent']:.1f}%")
print(f" 总查询数: {pool_stats['statistics']['total_queries']}")
# 获取节点状态
print("\n=== 节点状态 ===")
for node_id in list(impala.nodes.keys())[:2]: # 只显示前两个节点
hostname, port = node_id.split(':')
node_status = impala.get_node_status(hostname, int(port))
print(f"\n{node_status['node_id']}:")
print(f" 角色: {'协调器' if node_status['is_coordinator'] else ''}{'执行器' if node_status['is_executor'] else ''}")
print(f" 状态: {node_status['status']}")
print(f" 内存使用率: {node_status['memory']['utilization_percent']:.1f}%")
print(f" CPU使用率: {node_status['performance']['cpu_usage_percent']:.1f}%")
print(f" 当前查询数: {node_status['queries']['current']}")
print(f" 处理查询总数: {node_status['queries']['total_handled']}")
5. 数据流组件
5.1 Apache Kafka详解
from typing import Dict, List, Any, Optional, Tuple, Union, Callable
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
class KafkaMessageStatus(Enum):
"""Kafka消息状态"""
PENDING = "待发送"
SENT = "已发送"
ACKNOWLEDGED = "已确认"
FAILED = "发送失败"
EXPIRED = "已过期"
class KafkaPartitionState(Enum):
"""Kafka分区状态"""
ONLINE = "在线"
OFFLINE = "离线"
UNDER_REPLICATED = "副本不足"
LEADER_NOT_AVAILABLE = "Leader不可用"
class KafkaConsumerState(Enum):
"""Kafka消费者状态"""
ACTIVE = "活跃"
INACTIVE = "非活跃"
REBALANCING = "重平衡中"
DEAD = "已死亡"
class KafkaProducerAcks(Enum):
"""Kafka生产者确认模式"""
NONE = 0 # 不等待确认
LEADER = 1 # 等待Leader确认
ALL = -1 # 等待所有副本确认
@dataclass
class KafkaMessage:
"""Kafka消息"""
key: Optional[str]
value: str
headers: Dict[str, str]
timestamp: datetime
offset: Optional[int] = None
partition: Optional[int] = None
topic: Optional[str] = None
status: KafkaMessageStatus = KafkaMessageStatus.PENDING
retry_count: int = 0
error_message: Optional[str] = None
@dataclass
class KafkaPartition:
"""Kafka分区"""
topic: str
partition_id: int
leader: Optional[str]
replicas: List[str]
in_sync_replicas: List[str]
state: KafkaPartitionState
high_water_mark: int
log_end_offset: int
log_start_offset: int
messages: deque = field(default_factory=deque)
size_bytes: int = 0
last_updated: datetime = field(default_factory=datetime.now)
@dataclass
class KafkaTopic:
"""Kafka主题"""
name: str
partitions: Dict[int, KafkaPartition]
replication_factor: int
retention_ms: int
cleanup_policy: str
compression_type: str
max_message_bytes: int
created_at: datetime
config: Dict[str, Any] = field(default_factory=dict)
@dataclass
class KafkaConsumer:
"""Kafka消费者"""
consumer_id: str
group_id: str
client_id: str
subscribed_topics: List[str]
assigned_partitions: List[Tuple[str, int]] # (topic, partition)
state: KafkaConsumerState
last_heartbeat: datetime
session_timeout_ms: int
max_poll_interval_ms: int
auto_offset_reset: str
enable_auto_commit: bool
auto_commit_interval_ms: int
offsets: Dict[Tuple[str, int], int] # (topic, partition) -> offset
lag: Dict[Tuple[str, int], int] # (topic, partition) -> lag
@dataclass
class KafkaProducer:
"""Kafka生产者"""
producer_id: str
client_id: str
acks: KafkaProducerAcks
retries: int
batch_size: int
linger_ms: int
buffer_memory: int
compression_type: str
max_request_size: int
request_timeout_ms: int
delivery_timeout_ms: int
created_at: datetime
metrics: Dict[str, Any] = field(default_factory=dict)
@dataclass
class KafkaBroker:
"""Kafka代理节点"""
broker_id: int
hostname: str
port: int
rack: Optional[str]
is_controller: bool
state: str
start_time: datetime
last_heartbeat: datetime
disk_usage_bytes: int
network_in_rate: float
network_out_rate: float
request_rate: float
cpu_usage: float
memory_usage: float
active_connections: int
leader_partitions: List[Tuple[str, int]] # (topic, partition)
replica_partitions: List[Tuple[str, int]] # (topic, partition)
class KafkaCluster:
"""
Kafka集群管理器
"""
def __init__(self, cluster_id: str = "kafka-cluster"):
self.cluster_id = cluster_id
self.brokers = {}
self.topics = {}
self.consumers = {}
self.producers = {}
self.consumer_groups = defaultdict(list)
self.controller_broker_id = None
self.metadata_version = 1
self.cluster_metrics = {
'total_messages': 0,
'total_bytes': 0,
'messages_per_sec': 0.0,
'bytes_per_sec': 0.0,
'active_controllers': 0,
'under_replicated_partitions': 0,
'offline_partitions': 0
}
self.rebalance_lock = threading.Lock()
# 初始化默认代理
self._initialize_default_brokers()
def _initialize_default_brokers(self):
"""
初始化默认代理节点
"""
for i in range(3):
self.add_broker(
broker_id=i + 1,
hostname=f"kafka-broker-{i + 1}",
port=9092,
rack=f"rack-{i % 2 + 1}"
)
# 设置第一个代理为控制器
if self.brokers:
first_broker_id = min(self.brokers.keys())
self.controller_broker_id = first_broker_id
self.brokers[first_broker_id].is_controller = True
self.cluster_metrics['active_controllers'] = 1
def add_broker(self, broker_id: int, hostname: str, port: int = 9092,
rack: Optional[str] = None) -> Dict[str, Any]:
"""
添加Kafka代理节点
Args:
broker_id: 代理ID
hostname: 主机名
port: 端口
rack: 机架
Returns:
Dict[str, Any]: 添加结果
"""
if broker_id in self.brokers:
return {'status': 'error', 'message': f'Broker {broker_id} already exists'}
broker = KafkaBroker(
broker_id=broker_id,
hostname=hostname,
port=port,
rack=rack,
is_controller=False,
state="running",
start_time=datetime.now(),
last_heartbeat=datetime.now(),
disk_usage_bytes=random.randint(10 * 1024**3, 100 * 1024**3), # 10-100GB
network_in_rate=random.uniform(10.0, 100.0), # MB/s
network_out_rate=random.uniform(10.0, 100.0), # MB/s
request_rate=random.uniform(100.0, 1000.0), # requests/s
cpu_usage=random.uniform(10.0, 80.0), # %
memory_usage=random.uniform(20.0, 70.0), # %
active_connections=random.randint(50, 500),
leader_partitions=[],
replica_partitions=[]
)
self.brokers[broker_id] = broker
# 如果没有控制器，设置为控制器
if self.controller_broker_id is None:
self.controller_broker_id = broker_id
broker.is_controller = True
self.cluster_metrics['active_controllers'] = 1
return {
'status': 'success',
'broker_id': broker_id,
'hostname': hostname,
'port': port,
'is_controller': broker.is_controller
}
def remove_broker(self, broker_id: int) -> Dict[str, Any]:
"""
移除Kafka代理节点
Args:
broker_id: 代理ID
Returns:
Dict[str, Any]: 移除结果
"""
if broker_id not in self.brokers:
return {'status': 'error', 'message': f'Broker {broker_id} does not exist'}
broker = self.brokers[broker_id]
# 检查是否有分区在此代理上
leader_partitions = len(broker.leader_partitions)
replica_partitions = len(broker.replica_partitions)
if leader_partitions > 0 or replica_partitions > 0:
return {
'status': 'error',
'message': f'Broker {broker_id} has {leader_partitions} leader partitions and {replica_partitions} replica partitions'
}
# 如果是控制器，需要重新选举
was_controller = broker.is_controller
del self.brokers[broker_id]
if was_controller:
self._elect_new_controller()
return {
'status': 'success',
'broker_id': broker_id,
'was_controller': was_controller,
'leader_partitions': leader_partitions,
'replica_partitions': replica_partitions
}
def _elect_new_controller(self):
"""
选举新的控制器
"""
if not self.brokers:
self.controller_broker_id = None
self.cluster_metrics['active_controllers'] = 0
return
# 选择ID最小的代理作为新控制器
new_controller_id = min(self.brokers.keys())
# 清除旧控制器标记
for broker in self.brokers.values():
broker.is_controller = False
# 设置新控制器
self.brokers[new_controller_id].is_controller = True
self.controller_broker_id = new_controller_id
self.cluster_metrics['active_controllers'] = 1
def create_topic(self, topic_name: str, num_partitions: int = 3,
replication_factor: int = 2, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
创建主题
Args:
topic_name: 主题名称
num_partitions: 分区数
replication_factor: 副本因子
config: 主题配置
Returns:
Dict[str, Any]: 创建结果
"""
if topic_name in self.topics:
return {'status': 'error', 'message': f'Topic {topic_name} already exists'}
if len(self.brokers) < replication_factor:
return {
'status': 'error',
'message': f'Replication factor {replication_factor} is greater than available brokers {len(self.brokers)}'
}
# 默认配置
default_config = {
'retention.ms': 7 * 24 * 60 * 60 * 1000, # 7天
'cleanup.policy': 'delete',
'compression.type': 'producer',
'max.message.bytes': 1000000, # 1MB
'segment.ms': 7 * 24 * 60 * 60 * 1000, # 7天
'min.insync.replicas': max(1, replication_factor - 1)
}
if config:
default_config.update(config)
# 创建分区
partitions = {}
broker_ids = list(self.brokers.keys())
for partition_id in range(num_partitions):
# 选择Leader和副本
leader_broker = broker_ids[partition_id % len(broker_ids)]
replicas = [leader_broker]
# 添加其他副本
for i in range(1, replication_factor):
replica_broker = broker_ids[(partition_id + i) % len(broker_ids)]
if replica_broker not in replicas:
replicas.append(replica_broker)
partition = KafkaPartition(
topic=topic_name,
partition_id=partition_id,
leader=str(leader_broker),
replicas=[str(b) for b in replicas],
in_sync_replicas=[str(b) for b in replicas],
state=KafkaPartitionState.ONLINE,
high_water_mark=0,
log_end_offset=0,
log_start_offset=0
)
partitions[partition_id] = partition
# 更新代理的分区信息
for broker_id in replicas:
if broker_id == leader_broker:
self.brokers[broker_id].leader_partitions.append((topic_name, partition_id))
else:
self.brokers[broker_id].replica_partitions.append((topic_name, partition_id))
# 创建主题
topic = KafkaTopic(
name=topic_name,
partitions=partitions,
replication_factor=replication_factor,
retention_ms=default_config['retention.ms'],
cleanup_policy=default_config['cleanup.policy'],
compression_type=default_config['compression.type'],
max_message_bytes=default_config['max.message.bytes'],
created_at=datetime.now(),
config=default_config
)
self.topics[topic_name] = topic
self.metadata_version += 1
return {
'status': 'success',
'topic_name': topic_name,
'num_partitions': num_partitions,
'replication_factor': replication_factor,
'config': default_config
}
def delete_topic(self, topic_name: str) -> Dict[str, Any]:
"""
删除主题
Args:
topic_name: 主题名称
Returns:
Dict[str, Any]: 删除结果
"""
if topic_name not in self.topics:
return {'status': 'error', 'message': f'Topic {topic_name} does not exist'}
topic = self.topics[topic_name]
# 从代理中移除分区信息
for partition in topic.partitions.values():
for broker_id_str in partition.replicas:
broker_id = int(broker_id_str)
if broker_id in self.brokers:
broker = self.brokers[broker_id]
# 移除Leader分区
if (topic_name, partition.partition_id) in broker.leader_partitions:
broker.leader_partitions.remove((topic_name, partition.partition_id))
# 移除副本分区
if (topic_name, partition.partition_id) in broker.replica_partitions:
broker.replica_partitions.remove((topic_name, partition.partition_id))
# 移除消费者订阅
for consumer in self.consumers.values():
if topic_name in consumer.subscribed_topics:
consumer.subscribed_topics.remove(topic_name)
# 移除分配的分区
consumer.assigned_partitions = [
(t, p) for t, p in consumer.assigned_partitions if t != topic_name
]
# 移除偏移量
consumer.offsets = {
(t, p): offset for (t, p), offset in consumer.offsets.items() if t != topic_name
}
# 移除滞后信息
consumer.lag = {
(t, p): lag for (t, p), lag in consumer.lag.items() if t != topic_name
}
del self.topics[topic_name]
self.metadata_version += 1
return {
'status': 'success',
'topic_name': topic_name,
'partitions_deleted': len(topic.partitions),
'total_messages': sum(len(p.messages) for p in topic.partitions.values())
}
def create_producer(self, client_id: str, acks: KafkaProducerAcks = KafkaProducerAcks.ALL,
config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
创建生产者
Args:
client_id: 客户端ID
acks: 确认模式
config: 生产者配置
Returns:
Dict[str, Any]: 创建结果
"""
producer_id = f"producer-{len(self.producers) + 1}-{client_id}"
if producer_id in self.producers:
return {'status': 'error', 'message': f'Producer {producer_id} already exists'}
# 默认配置
default_config = {
'retries': 3,
'batch.size': 16384, # 16KB
'linger.ms': 0,
'buffer.memory': 33554432, # 32MB
'compression.type': 'none',
'max.request.size': 1048576, # 1MB
'request.timeout.ms': 30000,
'delivery.timeout.ms': 120000
}
if config:
default_config.update(config)
producer = KafkaProducer(
producer_id=producer_id,
client_id=client_id,
acks=acks,
retries=default_config['retries'],
batch_size=default_config['batch.size'],
linger_ms=default_config['linger.ms'],
buffer_memory=default_config['buffer.memory'],
compression_type=default_config['compression.type'],
max_request_size=default_config['max.request.size'],
request_timeout_ms=default_config['request.timeout.ms'],
delivery_timeout_ms=default_config['delivery.timeout.ms'],
created_at=datetime.now(),
metrics={
'messages_sent': 0,
'bytes_sent': 0,
'send_rate': 0.0,
'batch_size_avg': 0.0,
'record_queue_time_avg': 0.0,
'request_latency_avg': 0.0,
'errors': 0
}
)
self.producers[producer_id] = producer
return {
'status': 'success',
'producer_id': producer_id,
'client_id': client_id,
'acks': acks.value,
'config': default_config
}
def send_message(self, producer_id: str, topic: str, message: KafkaMessage,
partition: Optional[int] = None) -> Dict[str, Any]:
"""
发送消息
Args:
producer_id: 生产者ID
topic: 主题
message: 消息
partition: 指定分区
Returns:
Dict[str, Any]: 发送结果
"""
if producer_id not in self.producers:
return {'status': 'error', 'message': f'Producer {producer_id} not found'}
if topic not in self.topics:
return {'status': 'error', 'message': f'Topic {topic} not found'}
producer = self.producers[producer_id]
topic_obj = self.topics[topic]
# 选择分区
if partition is None:
if message.key:
# 基于key的哈希选择分区
partition = hash(message.key) % len(topic_obj.partitions)
else:
# 轮询选择分区
partition = random.randint(0, len(topic_obj.partitions) - 1)
if partition not in topic_obj.partitions:
return {'status': 'error', 'message': f'Partition {partition} not found in topic {topic}'}
partition_obj = topic_obj.partitions[partition]
# 检查分区状态
if partition_obj.state != KafkaPartitionState.ONLINE:
return {
'status': 'error',
'message': f'Partition {partition} is {partition_obj.state.value}'
}
# 检查消息大小
message_size = len(message.value.encode('utf-8'))
if message_size > topic_obj.max_message_bytes:
return {
'status': 'error',
'message': f'Message size {message_size} exceeds max {topic_obj.max_message_bytes}'
}
try:
# 设置消息属性
message.topic = topic
message.partition = partition
message.offset = partition_obj.log_end_offset
message.timestamp = datetime.now()
message.status = KafkaMessageStatus.SENT
# 添加到分区
partition_obj.messages.append(message)
partition_obj.log_end_offset += 1
partition_obj.high_water_mark = partition_obj.log_end_offset
partition_obj.size_bytes += message_size
partition_obj.last_updated = datetime.now()
# 更新生产者指标
producer.metrics['messages_sent'] += 1
producer.metrics['bytes_sent'] += message_size
# 更新集群指标
self.cluster_metrics['total_messages'] += 1
self.cluster_metrics['total_bytes'] += message_size
# 模拟确认
if producer.acks != KafkaProducerAcks.NONE:
message.status = KafkaMessageStatus.ACKNOWLEDGED
return {
'status': 'success',
'topic': topic,
'partition': partition,
'offset': message.offset,
'timestamp': message.timestamp.isoformat(),
'message_size': message_size
}
except Exception as e:
message.status = KafkaMessageStatus.FAILED
message.error_message = str(e)
producer.metrics['errors'] += 1
return {
'status': 'error',
'message': str(e),
'topic': topic,
'partition': partition
}
def create_consumer(self, group_id: str, client_id: str,
config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
创建消费者
Args:
group_id: 消费者组ID
client_id: 客户端ID
config: 消费者配置
Returns:
Dict[str, Any]: 创建结果
"""
consumer_id = f"consumer-{len(self.consumers) + 1}-{client_id}"
if consumer_id in self.consumers:
return {'status': 'error', 'message': f'Consumer {consumer_id} already exists'}
# 默认配置
default_config = {
'session.timeout.ms': 10000,
'max.poll.interval.ms': 300000,
'auto.offset.reset': 'latest',
'enable.auto.commit': True,
'auto.commit.interval.ms': 5000
}
if config:
default_config.update(config)
consumer = KafkaConsumer(
consumer_id=consumer_id,
group_id=group_id,
client_id=client_id,
subscribed_topics=[],
assigned_partitions=[],
state=KafkaConsumerState.INACTIVE,
last_heartbeat=datetime.now(),
session_timeout_ms=default_config['session.timeout.ms'],
max_poll_interval_ms=default_config['max.poll.interval.ms'],
auto_offset_reset=default_config['auto.offset.reset'],
enable_auto_commit=default_config['enable.auto.commit'],
auto_commit_interval_ms=default_config['auto.commit.interval.ms'],
offsets={},
lag={}
)
self.consumers[consumer_id] = consumer
self.consumer_groups[group_id].append(consumer_id)
return {
'status': 'success',
'consumer_id': consumer_id,
'group_id': group_id,
'client_id': client_id,
'config': default_config
}
def subscribe_topics(self, consumer_id: str, topics: List[str]) -> Dict[str, Any]:
"""
订阅主题
Args:
consumer_id: 消费者ID
topics: 主题列表
Returns:
Dict[str, Any]: 订阅结果
"""
if consumer_id not in self.consumers:
return {'status': 'error', 'message': f'Consumer {consumer_id} not found'}
consumer = self.consumers[consumer_id]
# 检查主题是否存在
missing_topics = [t for t in topics if t not in self.topics]
if missing_topics:
return {
'status': 'error',
'message': f'Topics not found: {missing_topics}'
}
consumer.subscribed_topics = topics
consumer.state = KafkaConsumerState.ACTIVE
# 触发重平衡
self._trigger_rebalance(consumer.group_id)
return {
'status': 'success',
'consumer_id': consumer_id,
'subscribed_topics': topics,
'assigned_partitions': consumer.assigned_partitions
}
def _trigger_rebalance(self, group_id: str):
"""
触发消费者组重平衡
Args:
group_id: 消费者组ID
"""
with self.rebalance_lock:
group_consumers = [self.consumers[cid] for cid in self.consumer_groups[group_id]
if cid in self.consumers and self.consumers[cid].state == KafkaConsumerState.ACTIVE]
if not group_consumers:
return
# 设置重平衡状态
for consumer in group_consumers:
consumer.state = KafkaConsumerState.REBALANCING
# 收集所有订阅的主题和分区
all_partitions = []
for consumer in group_consumers:
for topic in consumer.subscribed_topics:
if topic in self.topics:
for partition_id in self.topics[topic].partitions.keys():
all_partitions.append((topic, partition_id))
# 去重
all_partitions = list(set(all_partitions))
# 重新分配分区（简单轮询策略）
for i, consumer in enumerate(group_consumers):
consumer.assigned_partitions = []
consumer.offsets = {}
consumer.lag = {}
for i, partition in enumerate(all_partitions):
consumer_index = i % len(group_consumers)
consumer = group_consumers[consumer_index]
consumer.assigned_partitions.append(partition)
# 初始化偏移量
topic, partition_id = partition
if consumer.auto_offset_reset == 'earliest':
offset = self.topics[topic].partitions[partition_id].log_start_offset
else: # latest
offset = self.topics[topic].partitions[partition_id].log_end_offset
consumer.offsets[partition] = offset
# 计算滞后
current_offset = self.topics[topic].partitions[partition_id].log_end_offset
consumer.lag[partition] = max(0, current_offset - offset)
# 恢复活跃状态
for consumer in group_consumers:
consumer.state = KafkaConsumerState.ACTIVE
def poll_messages(self, consumer_id: str, max_records: int = 100,
timeout_ms: int = 1000) -> Dict[str, Any]:
"""
拉取消息
Args:
consumer_id: 消费者ID
max_records: 最大记录数
timeout_ms: 超时时间
Returns:
Dict[str, Any]: 拉取结果
"""
if consumer_id not in self.consumers:
return {'status': 'error', 'message': f'Consumer {consumer_id} not found'}
consumer = self.consumers[consumer_id]
if consumer.state != KafkaConsumerState.ACTIVE:
return {
'status': 'error',
'message': f'Consumer {consumer_id} is not active (state: {consumer.state.value})'
}
messages = []
records_fetched = 0
for topic, partition_id in consumer.assigned_partitions:
if records_fetched >= max_records:
break
if topic not in self.topics:
continue
partition = self.topics[topic].partitions[partition_id]
current_offset = consumer.offsets.get((topic, partition_id), 0)
# 获取消息
for message in partition.messages:
if message.offset >= current_offset and records_fetched < max_records:
messages.append({
'topic': topic,
'partition': partition_id,
'offset': message.offset,
'key': message.key,
'value': message.value,
'headers': message.headers,
'timestamp': message.timestamp.isoformat()
})
records_fetched += 1
# 更新偏移量
consumer.offsets[(topic, partition_id)] = message.offset + 1
# 更新滞后信息
for topic, partition_id in consumer.assigned_partitions:
if topic in self.topics:
current_offset = consumer.offsets.get((topic, partition_id), 0)
latest_offset = self.topics[topic].partitions[partition_id].log_end_offset
consumer.lag[(topic, partition_id)] = max(0, latest_offset - current_offset)
# 更新心跳
consumer.last_heartbeat = datetime.now()
return {
'status': 'success',
'consumer_id': consumer_id,
'messages': messages,
'records_fetched': records_fetched,
'has_more': any(
consumer.offsets.get((t, p), 0) < self.topics[t].partitions[p].log_end_offset
for t, p in consumer.assigned_partitions if t in self.topics
)
}
def commit_offsets(self, consumer_id: str, offsets: Optional[Dict[Tuple[str, int], int]] = None) -> Dict[str, Any]:
"""
提交偏移量
Args:
consumer_id: 消费者ID
offsets: 偏移量字典，如果为None则提交当前偏移量
Returns:
Dict[str, Any]: 提交结果
"""
if consumer_id not in self.consumers:
return {'status': 'error', 'message': f'Consumer {consumer_id} not found'}
consumer = self.consumers[consumer_id]
if offsets is None:
offsets = consumer.offsets.copy()
committed_offsets = {}
for (topic, partition_id), offset in offsets.items():
if (topic, partition_id) in consumer.assigned_partitions:
consumer.offsets[(topic, partition_id)] = offset
committed_offsets[(topic, partition_id)] = offset
return {
'status': 'success',
'consumer_id': consumer_id,
'committed_offsets': {f"{t}-{p}": o for (t, p), o in committed_offsets.items()}
}
def get_cluster_metadata(self) -> Dict[str, Any]:
"""
获取集群元数据
Returns:
Dict[str, Any]: 集群元数据
"""
# 统计分区状态
total_partitions = 0
online_partitions = 0
under_replicated_partitions = 0
offline_partitions = 0
for topic in self.topics.values():
for partition in topic.partitions.values():
total_partitions += 1
if partition.state == KafkaPartitionState.ONLINE:
online_partitions += 1
elif partition.state == KafkaPartitionState.UNDER_REPLICATED:
under_replicated_partitions += 1
elif partition.state == KafkaPartitionState.OFFLINE:
offline_partitions += 1
# 更新集群指标
self.cluster_metrics['under_replicated_partitions'] = under_replicated_partitions
self.cluster_metrics['offline_partitions'] = offline_partitions
# 统计消费者组
consumer_group_stats = {}
for group_id, consumer_ids in self.consumer_groups.items():
active_consumers = sum(1 for cid in consumer_ids
if cid in self.consumers and self.consumers[cid].state == KafkaConsumerState.ACTIVE)
total_lag = sum(
sum(self.consumers[cid].lag.values()) for cid in consumer_ids
if cid in self.consumers
)
consumer_group_stats[group_id] = {
'total_consumers': len(consumer_ids),
'active_consumers': active_consumers,
'total_lag': total_lag
}
return {
'cluster_id': self.cluster_id,
'metadata_version': self.metadata_version,
'controller_broker_id': self.controller_broker_id,
'brokers': {
'total': len(self.brokers),
'online': sum(1 for b in self.brokers.values() if b.state == "running"),
'controllers': sum(1 for b in self.brokers.values() if b.is_controller)
},
'topics': {
'total': len(self.topics),
'names': list(self.topics.keys())
},
'partitions': {
'total': total_partitions,
'online': online_partitions,
'under_replicated': under_replicated_partitions,
'offline': offline_partitions
},
'producers': {
'total': len(self.producers),
'active': len(self.producers) # 简化：假设所有生产者都活跃
},
'consumer_groups': {
'total': len(self.consumer_groups),
'stats': consumer_group_stats
},
'cluster_metrics': self.cluster_metrics,
'timestamp': datetime.now().isoformat()
}
def get_topic_info(self, topic_name: str) -> Dict[str, Any]:
"""
获取主题信息
Args:
topic_name: 主题名称
Returns:
Dict[str, Any]: 主题信息
"""
if topic_name not in self.topics:
return {'status': 'error', 'message': f'Topic {topic_name} not found'}
topic = self.topics[topic_name]
# 统计分区信息
partition_info = []
total_messages = 0
total_size = 0
for partition_id, partition in topic.partitions.items():
partition_messages = len(partition.messages)
total_messages += partition_messages
total_size += partition.size_bytes
partition_info.append({
'partition_id': partition_id,
'leader': partition.leader,
'replicas': partition.replicas,
'in_sync_replicas': partition.in_sync_replicas,
'state': partition.state.value,
'high_water_mark': partition.high_water_mark,
'log_end_offset': partition.log_end_offset,
'log_start_offset': partition.log_start_offset,
'messages': partition_messages,
'size_bytes': partition.size_bytes,
'last_updated': partition.last_updated.isoformat()
})
return {
'topic_name': topic.name,
'num_partitions': len(topic.partitions),
'replication_factor': topic.replication_factor,
'retention_ms': topic.retention_ms,
'cleanup_policy': topic.cleanup_policy,
'compression_type': topic.compression_type,
'max_message_bytes': topic.max_message_bytes,
'created_at': topic.created_at.isoformat(),
'config': topic.config,
'statistics': {
'total_messages': total_messages,
'total_size_bytes': total_size,
'avg_message_size': total_size / total_messages if total_messages > 0 else 0
},
'partitions': partition_info
}
def get_consumer_group_info(self, group_id: str) -> Dict[str, Any]:
"""
获取消费者组信息
Args:
group_id: 消费者组ID
Returns:
Dict[str, Any]: 消费者组信息
"""
if group_id not in self.consumer_groups:
return {'status': 'error', 'message': f'Consumer group {group_id} not found'}
consumer_ids = self.consumer_groups[group_id]
consumers_info = []
total_lag = 0
for consumer_id in consumer_ids:
if consumer_id in self.consumers:
consumer = self.consumers[consumer_id]
consumer_lag = sum(consumer.lag.values())
total_lag += consumer_lag
consumers_info.append({
'consumer_id': consumer.consumer_id,
'client_id': consumer.client_id,
'state': consumer.state.value,
'subscribed_topics': consumer.subscribed_topics,
'assigned_partitions': [f"{t}-{p}" for t, p in consumer.assigned_partitions],
'last_heartbeat': consumer.last_heartbeat.isoformat(),
'lag': consumer_lag,
'offsets': {f"{t}-{p}": o for (t, p), o in consumer.offsets.items()}
})
return {
'group_id': group_id,
'total_consumers': len(consumer_ids),
'active_consumers': sum(1 for cid in consumer_ids
if cid in self.consumers and self.consumers[cid].state == KafkaConsumerState.ACTIVE),
'total_lag': total_lag,
'consumers': consumers_info
}
# 使用示例
if __name__ == "__main__":
# 创建Kafka集群
kafka = KafkaCluster("production-kafka")
print("=== Kafka集群管理示例 ===")
# 创建主题
print("\n=== 创建主题 ===")
topic_result = kafka.create_topic(
topic_name="user-events",
num_partitions=6,
replication_factor=2,
config={
'retention.ms': 24 * 60 * 60 * 1000, # 1天
'cleanup.policy': 'delete',
'compression.type': 'gzip'
}
)
print(f"主题创建结果: {topic_result}")
# 创建生产者
print("\n=== 创建生产者 ===")
producer_result = kafka.create_producer(
client_id="event-producer",
acks=KafkaProducerAcks.ALL,
config={'batch.size': 32768, 'linger.ms': 10}
)
print(f"生产者创建结果: {producer_result}")
producer_id = producer_result.get('producer_id')
# 发送消息
print("\n=== 发送消息 ===")
for i in range(10):
message = KafkaMessage(
key=f"user-{i % 3}",
value=json.dumps({
'user_id': i % 3,
'event_type': 'click',
'timestamp': datetime.now().isoformat(),
'page': f'/page-{i % 5}'
}),
headers={'source': 'web-app', 'version': '1.0'}
)
send_result = kafka.send_message(producer_id, "user-events", message)
if i < 3: # 只显示前3条
print(f"消息发送结果: {send_result}")
# 创建消费者
print("\n=== 创建消费者 ===")
consumer_result = kafka.create_consumer(
group_id="analytics-group",
client_id="analytics-consumer",
config={'auto.offset.reset': 'earliest'}
)
print(f"消费者创建结果: {consumer_result}")
consumer_id = consumer_result.get('consumer_id')
# 订阅主题
print("\n=== 订阅主题 ===")
subscribe_result = kafka.subscribe_topics(consumer_id, ["user-events"])
print(f"订阅结果: {subscribe_result}")
# 拉取消息
print("\n=== 拉取消息 ===")
poll_result = kafka.poll_messages(consumer_id, max_records=5)
print(f"拉取到 {poll_result['records_fetched']} 条消息")
for msg in poll_result['messages'][:3]: # 只显示前3条
print(f" 消息: {msg['key']} -> {msg['value'][:50]}...")
# 提交偏移量
print("\n=== 提交偏移量 ===")
commit_result = kafka.commit_offsets(consumer_id)
print(f"偏移量提交结果: {commit_result}")
# 获取集群元数据
print("\n=== 集群元数据 ===")
metadata = kafka.get_cluster_metadata()
print(f"集群ID: {metadata['cluster_id']}")
print(f"代理数量: {metadata['brokers']['total']}")
print(f"主题数量: {metadata['topics']['total']}")
print(f"分区总数: {metadata['partitions']['total']}")
print(f"消费者组数: {metadata['consumer_groups']['total']}")
print(f"总消息数: {metadata['cluster_metrics']['total_messages']}")
# 获取主题信息
print("\n=== 主题信息 ===")
topic_info = kafka.get_topic_info("user-events")
print(f"主题: {topic_info['topic_name']}")
print(f"分区数: {topic_info['num_partitions']}")
print(f"副本因子: {topic_info['replication_factor']}")
print(f"总消息数: {topic_info['statistics']['total_messages']}")
print(f"总大小: {topic_info['statistics']['total_size_bytes']} bytes")
# 获取消费者组信息
print("\n=== 消费者组信息 ===")
group_info = kafka.get_consumer_group_info("analytics-group")
print(f"消费者组: {group_info['group_id']}")
print(f"消费者总数: {group_info['total_consumers']}")
print(f"活跃消费者: {group_info['active_consumers']}")
print(f"总滞后: {group_info['total_lag']}")
5.2 Apache ZooKeeper详解
from typing import Dict, List, Any, Optional, Tuple, Union, Callable
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import hashlib
class ZNodeType(Enum):
"""ZNode类型"""
PERSISTENT = "持久节点"
EPHEMERAL = "临时节点"
PERSISTENT_SEQUENTIAL = "持久顺序节点"
EPHEMERAL_SEQUENTIAL = "临时顺序节点"
CONTAINER = "容器节点"
PERSISTENT_TTL = "持久TTL节点"
class ZNodeState(Enum):
"""ZNode状态"""
ACTIVE = "活跃"
DELETED = "已删除"
EXPIRED = "已过期"
class ZooKeeperState(Enum):
"""ZooKeeper服务器状态"""
LOOKING = "寻找Leader"
FOLLOWING = "跟随者"
LEADING = "领导者"
OBSERVING = "观察者"
class WatcherType(Enum):
"""监听器类型"""
NODE_CREATED = "节点创建"
NODE_DELETED = "节点删除"
NODE_DATA_CHANGED = "节点数据变更"
NODE_CHILDREN_CHANGED = "子节点变更"
CONNECTION_STATE_CHANGED = "连接状态变更"
@dataclass
class ZNodeStat:
"""ZNode统计信息"""
czxid: int # 创建事务ID
mzxid: int # 最后修改事务ID
pzxid: int # 最后修改子节点事务ID
ctime: datetime # 创建时间
mtime: datetime # 最后修改时间
version: int # 数据版本
cversion: int # 子节点版本
aversion: int # ACL版本
ephemeral_owner: int # 临时节点所有者会话ID
data_length: int # 数据长度
num_children: int # 子节点数量
@dataclass
class ZNode:
"""ZooKeeper节点"""
path: str
data: bytes
node_type: ZNodeType
state: ZNodeState
stat: ZNodeStat
children: Dict[str, 'ZNode'] = field(default_factory=dict)
watchers: List['Watcher'] = field(default_factory=list)
acl: List[Dict[str, Any]] = field(default_factory=list)
ttl: Optional[int] = None # TTL毫秒
session_id: Optional[int] = None # 临时节点的会话ID
@dataclass
class Watcher:
"""监听器"""
watcher_id: str
session_id: int
path: str
watcher_type: WatcherType
callback: Optional[Callable] = None
created_at: datetime = field(default_factory=datetime.now)
triggered: bool = False
@dataclass
class ZooKeeperSession:
"""ZooKeeper会话"""
session_id: int
client_id: str
timeout: int # 会话超时时间（毫秒）
created_at: datetime
last_seen: datetime
state: str
ephemeral_nodes: List[str] = field(default_factory=list)
watchers: List[str] = field(default_factory=list)
@dataclass
class ZooKeeperServer:
"""ZooKeeper服务器"""
server_id: int
hostname: str
port: int
state: ZooKeeperState
last_zxid: int # 最后事务ID
epoch: int # 选举轮次
votes_received: int
start_time: datetime
last_heartbeat: datetime
is_leader: bool = False
followers: List[int] = field(default_factory=list)
pending_proposals: List[Dict[str, Any]] = field(default_factory=list)
class ZooKeeperCluster:
"""
ZooKeeper集群管理器
"""
def __init__(self, cluster_id: str = "zk-cluster"):
self.cluster_id = cluster_id
self.servers = {}
self.sessions = {}
self.root_node = self._create_root_node()
self.current_zxid = 1000
self.leader_id = None
self.election_lock = threading.Lock()
self.session_lock = threading.Lock()
self.node_lock = threading.Lock()
self.next_session_id = 1
self.watchers = {}
# 初始化默认服务器
self._initialize_default_servers()
# 启动选举
self._start_leader_election()
def _create_root_node(self) -> ZNode:
"""创建根节点"""
stat = ZNodeStat(
czxid=self.current_zxid,
mzxid=self.current_zxid,
pzxid=self.current_zxid,
ctime=datetime.now(),
mtime=datetime.now(),
version=0,
cversion=0,
aversion=0,
ephemeral_owner=0,
data_length=0,
num_children=0
)
return ZNode(
path="/",
data=b"",
node_type=ZNodeType.PERSISTENT,
state=ZNodeState.ACTIVE,
stat=stat
)
def _initialize_default_servers(self):
"""初始化默认服务器"""
for i in range(3):
self.add_server(
server_id=i + 1,
hostname=f"zk-server-{i + 1}",
port=2181
)
def add_server(self, server_id: int, hostname: str, port: int = 2181) -> Dict[str, Any]:
"""
添加ZooKeeper服务器
Args:
server_id: 服务器ID
hostname: 主机名
port: 端口
Returns:
Dict[str, Any]: 添加结果
"""
if server_id in self.servers:
return {'status': 'error', 'message': f'Server {server_id} already exists'}
server = ZooKeeperServer(
server_id=server_id,
hostname=hostname,
port=port,
state=ZooKeeperState.LOOKING,
last_zxid=self.current_zxid,
epoch=0,
votes_received=0,
start_time=datetime.now(),
last_heartbeat=datetime.now()
)
self.servers[server_id] = server
return {
'status': 'success',
'server_id': server_id,
'hostname': hostname,
'port': port,
'state': server.state.value
}
def _start_leader_election(self):
"""启动Leader选举"""
with self.election_lock:
if len(self.servers) == 0:
return
# 简化的选举算法：选择ID最大的服务器作为Leader
candidate_id = max(self.servers.keys())
# 重置所有服务器状态
for server in self.servers.values():
server.state = ZooKeeperState.LOOKING
server.is_leader = False
server.followers = []
server.votes_received = 0
server.epoch += 1
# 设置Leader
leader = self.servers[candidate_id]
leader.state = ZooKeeperState.LEADING
leader.is_leader = True
self.leader_id = candidate_id
# 设置Followers
for server_id, server in self.servers.items():
if server_id != candidate_id:
server.state = ZooKeeperState.FOLLOWING
leader.followers.append(server_id)
def create_session(self, client_id: str, timeout: int = 30000) -> Dict[str, Any]:
"""
创建客户端会话
Args:
client_id: 客户端ID
timeout: 会话超时时间（毫秒）
Returns:
Dict[str, Any]: 会话创建结果
"""
with self.session_lock:
session_id = self.next_session_id
self.next_session_id += 1
session = ZooKeeperSession(
session_id=session_id,
client_id=client_id,
timeout=timeout,
created_at=datetime.now(),
last_seen=datetime.now(),
state="connected"
)
self.sessions[session_id] = session
return {
'status': 'success',
'session_id': session_id,
'timeout': timeout,
'client_id': client_id
}
def close_session(self, session_id: int) -> Dict[str, Any]:
"""
关闭客户端会话
Args:
session_id: 会话ID
Returns:
Dict[str, Any]: 关闭结果
"""
if session_id not in self.sessions:
return {'status': 'error', 'message': f'Session {session_id} not found'}
session = self.sessions[session_id]
# 删除临时节点
deleted_nodes = []
for node_path in session.ephemeral_nodes.copy():
delete_result = self.delete_node(node_path, session_id=session_id)
if delete_result['status'] == 'success':
deleted_nodes.append(node_path)
# 移除监听器
removed_watchers = []
for watcher_id in session.watchers.copy():
if watcher_id in self.watchers:
del self.watchers[watcher_id]
removed_watchers.append(watcher_id)
# 删除会话
del self.sessions[session_id]
return {
'status': 'success',
'session_id': session_id,
'deleted_nodes': deleted_nodes,
'removed_watchers': removed_watchers
}
def create_node(self, path: str, data: bytes = b"", node_type: ZNodeType = ZNodeType.PERSISTENT,
session_id: Optional[int] = None, ttl: Optional[int] = None) -> Dict[str, Any]:
"""
创建ZNode
Args:
path: 节点路径
data: 节点数据
node_type: 节点类型
session_id: 会话ID（临时节点需要）
ttl: TTL时间（毫秒）
Returns:
Dict[str, Any]: 创建结果
"""
with self.node_lock:
# 验证路径
if not path.startswith('/'):
return {'status': 'error', 'message': 'Path must start with /'}
# 检查父节点是否存在
parent_path = '/'.join(path.split('/')[:-1]) or '/'
parent_node = self._find_node(parent_path)
if not parent_node:
return {'status': 'error', 'message': f'Parent node {parent_path} does not exist'}
# 处理顺序节点
actual_path = path
if node_type in [ZNodeType.PERSISTENT_SEQUENTIAL, ZNodeType.EPHEMERAL_SEQUENTIAL]:
sequence_num = parent_node.stat.cversion
actual_path = f"{path}{sequence_num:010d}"
# 检查节点是否已存在
if self._find_node(actual_path):
return {'status': 'error', 'message': f'Node {actual_path} already exists'}
# 验证临时节点的会话
if node_type in [ZNodeType.EPHEMERAL, ZNodeType.EPHEMERAL_SEQUENTIAL]:
if not session_id or session_id not in self.sessions:
return {'status': 'error', 'message': 'Ephemeral node requires valid session'}
# 创建节点统计信息
self.current_zxid += 1
stat = ZNodeStat(
czxid=self.current_zxid,
mzxid=self.current_zxid,
pzxid=self.current_zxid,
ctime=datetime.now(),
mtime=datetime.now(),
version=0,
cversion=0,
aversion=0,
ephemeral_owner=session_id if node_type in [ZNodeType.EPHEMERAL, ZNodeType.EPHEMERAL_SEQUENTIAL] else 0,
data_length=len(data),
num_children=0
)
# 创建节点
node = ZNode(
path=actual_path,
data=data,
node_type=node_type,
state=ZNodeState.ACTIVE,
stat=stat,
ttl=ttl,
session_id=session_id if node_type in [ZNodeType.EPHEMERAL, ZNodeType.EPHEMERAL_SEQUENTIAL] else None
)
# 添加到父节点
node_name = actual_path.split('/')[-1]
parent_node.children[node_name] = node
parent_node.stat.num_children += 1
parent_node.stat.cversion += 1
parent_node.stat.pzxid = self.current_zxid
# 如果是临时节点，添加到会话
if session_id and node_type in [ZNodeType.EPHEMERAL, ZNodeType.EPHEMERAL_SEQUENTIAL]:
self.sessions[session_id].ephemeral_nodes.append(actual_path)
# 触发监听器
self._trigger_watchers(parent_path, WatcherType.NODE_CHILDREN_CHANGED)
self._trigger_watchers(actual_path, WatcherType.NODE_CREATED)
return {
'status': 'success',
'path': actual_path,
'zxid': self.current_zxid,
'stat': {
'czxid': stat.czxid,
'mzxid': stat.mzxid,
'ctime': stat.ctime.isoformat(),
'mtime': stat.mtime.isoformat(),
'version': stat.version,
'data_length': stat.data_length,
'num_children': stat.num_children
}
}
def delete_node(self, path: str, version: int = -1, session_id: Optional[int] = None) -> Dict[str, Any]:
"""
删除ZNode
Args:
path: 节点路径
version: 期望版本（-1表示忽略版本）
session_id: 会话ID
Returns:
Dict[str, Any]: 删除结果
"""
with self.node_lock:
node = self._find_node(path)
if not node:
return {'status': 'error', 'message': f'Node {path} does not exist'}
# 检查版本
if version != -1 and node.stat.version != version:
return {
'status': 'error',
'message': f'Version mismatch: expected {version}, got {node.stat.version}'
}
# 检查是否有子节点
if node.children:
return {'status': 'error', 'message': f'Node {path} has children'}
# 检查临时节点的会话权限
if node.session_id and node.session_id != session_id:
return {'status': 'error', 'message': 'Cannot delete ephemeral node from different session'}
# 从父节点移除
parent_path = '/'.join(path.split('/')[:-1]) or '/'
parent_node = self._find_node(parent_path)
if parent_node:
node_name = path.split('/')[-1]
if node_name in parent_node.children:
del parent_node.children[node_name]
parent_node.stat.num_children -= 1
parent_node.stat.cversion += 1
self.current_zxid += 1
parent_node.stat.pzxid = self.current_zxid
# 从会话中移除临时节点
if node.session_id and node.session_id in self.sessions:
session = self.sessions[node.session_id]
if path in session.ephemeral_nodes:
session.ephemeral_nodes.remove(path)
# 标记为已删除
node.state = ZNodeState.DELETED
# 触发监听器
self._trigger_watchers(path, WatcherType.NODE_DELETED)
self._trigger_watchers(parent_path, WatcherType.NODE_CHILDREN_CHANGED)
return {
'status': 'success',
'path': path,
'zxid': self.current_zxid
}
def get_node(self, path: str, watch: bool = False, session_id: Optional[int] = None) -> Dict[str, Any]:
"""
获取ZNode数据
Args:
path: 节点路径
watch: 是否设置监听器
session_id: 会话ID
Returns:
Dict[str, Any]: 节点数据
"""
node = self._find_node(path)
if not node or node.state != ZNodeState.ACTIVE:
return {'status': 'error', 'message': f'Node {path} does not exist'}
# 设置监听器
if watch and session_id:
self._add_watcher(path, WatcherType.NODE_DATA_CHANGED, session_id)
return {
'status': 'success',
'path': path,
'data': node.data.decode('utf-8', errors='ignore'),
'stat': {
'czxid': node.stat.czxid,
'mzxid': node.stat.mzxid,
'pzxid': node.stat.pzxid,
'ctime': node.stat.ctime.isoformat(),
'mtime': node.stat.mtime.isoformat(),
'version': node.stat.version,
'cversion': node.stat.cversion,
'aversion': node.stat.aversion,
'ephemeral_owner': node.stat.ephemeral_owner,
'data_length': node.stat.data_length,
'num_children': node.stat.num_children
}
}
def set_node_data(self, path: str, data: bytes, version: int = -1) -> Dict[str, Any]:
"""
设置ZNode数据
Args:
path: 节点路径
data: 新数据
version: 期望版本（-1表示忽略版本）
Returns:
Dict[str, Any]: 设置结果
"""
with self.node_lock:
node = self._find_node(path)
if not node or node.state != ZNodeState.ACTIVE:
return {'status': 'error', 'message': f'Node {path} does not exist'}
# 检查版本
if version != -1 and node.stat.version != version:
return {
'status': 'error',
'message': f'Version mismatch: expected {version}, got {node.stat.version}'
}
# 更新数据
node.data = data
node.stat.version += 1
self.current_zxid += 1
node.stat.mzxid = self.current_zxid
node.stat.mtime = datetime.now()
node.stat.data_length = len(data)
# 触发监听器
self._trigger_watchers(path, WatcherType.NODE_DATA_CHANGED)
return {
'status': 'success',
'path': path,
'zxid': self.current_zxid,
'stat': {
'mzxid': node.stat.mzxid,
'mtime': node.stat.mtime.isoformat(),
'version': node.stat.version,
'data_length': node.stat.data_length
}
}
def get_children(self, path: str, watch: bool = False, session_id: Optional[int] = None) -> Dict[str, Any]:
"""
获取子节点列表
Args:
path: 节点路径
watch: 是否设置监听器
session_id: 会话ID
Returns:
Dict[str, Any]: 子节点列表
"""
node = self._find_node(path)
if not node or node.state != ZNodeState.ACTIVE:
return {'status': 'error', 'message': f'Node {path} does not exist'}
# 设置监听器
if watch and session_id:
self._add_watcher(path, WatcherType.NODE_CHILDREN_CHANGED, session_id)
children = [name for name, child in node.children.items() if child.state == ZNodeState.ACTIVE]
children.sort() # ZooKeeper返回排序的子节点列表
return {
'status': 'success',
'path': path,
'children': children,
'stat': {
'cversion': node.stat.cversion,
'num_children': len(children)
}
}
def exists(self, path: str, watch: bool = False, session_id: Optional[int] = None) -> Dict[str, Any]:
"""
检查节点是否存在
Args:
path: 节点路径
watch: 是否设置监听器
session_id: 会话ID
Returns:
Dict[str, Any]: 存在性检查结果
"""
node = self._find_node(path)
exists = node is not None and node.state == ZNodeState.ACTIVE
# 设置监听器（即使节点不存在也可以监听创建事件）
if watch and session_id:
if exists:
self._add_watcher(path, WatcherType.NODE_DATA_CHANGED, session_id)
else:
self._add_watcher(path, WatcherType.NODE_CREATED, session_id)
result = {
'status': 'success',
'path': path,
'exists': exists
}
if exists:
result['stat'] = {
'czxid': node.stat.czxid,
'mzxid': node.stat.mzxid,
'ctime': node.stat.ctime.isoformat(),
'mtime': node.stat.mtime.isoformat(),
'version': node.stat.version,
'data_length': node.stat.data_length,
'num_children': node.stat.num_children
}
return result
def _find_node(self, path: str) -> Optional[ZNode]:
"""
查找节点
Args:
path: 节点路径
Returns:
Optional[ZNode]: 找到的节点或None
"""
if path == '/':
return self.root_node
parts = [p for p in path.split('/') if p]
current = self.root_node
for part in parts:
if part not in current.children:
return None
current = current.children[part]
if current.state != ZNodeState.ACTIVE:
return None
return current
def _add_watcher(self, path: str, watcher_type: WatcherType, session_id: int):
"""
添加监听器
Args:
path: 监听路径
watcher_type: 监听器类型
session_id: 会话ID
"""
if session_id not in self.sessions:
return
watcher_id = f"watcher-{len(self.watchers) + 1}-{session_id}"
watcher = Watcher(
watcher_id=watcher_id,
session_id=session_id,
path=path,
watcher_type=watcher_type
)
self.watchers[watcher_id] = watcher
self.sessions[session_id].watchers.append(watcher_id)
def _trigger_watchers(self, path: str, event_type: WatcherType):
"""
触发监听器
Args:
path: 事件路径
event_type: 事件类型
"""
triggered_watchers = []
for watcher_id, watcher in list(self.watchers.items()):
if watcher.path == path and watcher.watcher_type == event_type and not watcher.triggered:
watcher.triggered = True
triggered_watchers.append(watcher_id)
# 从会话中移除监听器
if watcher.session_id in self.sessions:
session = self.sessions[watcher.session_id]
if watcher_id in session.watchers:
session.watchers.remove(watcher_id)
# 删除监听器（一次性触发）
del self.watchers[watcher_id]
return triggered_watchers
def get_cluster_state(self) -> Dict[str, Any]:
"""
获取集群状态
Returns:
Dict[str, Any]: 集群状态
"""
# 统计节点
def count_nodes(node: ZNode) -> Tuple[int, int]:
active_count = 1 if node.state == ZNodeState.ACTIVE else 0
total_count = 1
for child in node.children.values():
child_active, child_total = count_nodes(child)
active_count += child_active
total_count += child_total
return active_count, total_count
active_nodes, total_nodes = count_nodes(self.root_node)
# 统计会话
active_sessions = sum(1 for s in self.sessions.values() if s.state == "connected")
# 统计临时节点
ephemeral_nodes = sum(len(s.ephemeral_nodes) for s in self.sessions.values())
# 统计监听器
active_watchers = len(self.watchers)
return {
'cluster_id': self.cluster_id,
'leader_id': self.leader_id,
'current_zxid': self.current_zxid,
'servers': {
'total': len(self.servers),
'leader': 1 if self.leader_id else 0,
'followers': len([s for s in self.servers.values() if s.state == ZooKeeperState.FOLLOWING]),
'looking': len([s for s in self.servers.values() if s.state == ZooKeeperState.LOOKING])
},
'nodes': {
'total': total_nodes,
'active': active_nodes,
'ephemeral': ephemeral_nodes
},
'sessions': {
'total': len(self.sessions),
'active': active_sessions
},
'watchers': {
'active': active_watchers
},
'timestamp': datetime.now().isoformat()
}
def get_server_info(self, server_id: int) -> Dict[str, Any]:
"""
获取服务器信息
Args:
server_id: 服务器ID
Returns:
Dict[str, Any]: 服务器信息
"""
if server_id not in self.servers:
return {'status': 'error', 'message': f'Server {server_id} not found'}
server = self.servers[server_id]
return {
'server_id': server.server_id,
'hostname': server.hostname,
'port': server.port,
'state': server.state.value,
'is_leader': server.is_leader,
'last_zxid': server.last_zxid,
'epoch': server.epoch,
'start_time': server.start_time.isoformat(),
'last_heartbeat': server.last_heartbeat.isoformat(),
'followers': server.followers if server.is_leader else [],
'pending_proposals': len(server.pending_proposals)
}
def get_session_info(self, session_id: int) -> Dict[str, Any]:
"""
获取会话信息
Args:
session_id: 会话ID
Returns:
Dict[str, Any]: 会话信息
"""
if session_id not in self.sessions:
return {'status': 'error', 'message': f'Session {session_id} not found'}
session = self.sessions[session_id]
return {
'session_id': session.session_id,
'client_id': session.client_id,
'timeout': session.timeout,
'state': session.state,
'created_at': session.created_at.isoformat(),
'last_seen': session.last_seen.isoformat(),
'ephemeral_nodes': session.ephemeral_nodes,
'watchers': len(session.watchers)
}
# 使用示例
if __name__ == "__main__":
# 创建ZooKeeper集群
zk = ZooKeeperCluster("production-zk")
print("=== ZooKeeper集群管理示例 ===")
# 创建会话
print("\n=== 创建会话 ===")
session_result = zk.create_session("app-client-1", timeout=30000)
print(f"会话创建结果: {session_result}")
session_id = session_result.get('session_id')
# 创建节点
print("\n=== 创建节点 ===")
# 创建持久节点
create_result = zk.create_node(
"/app",
b"application config",
ZNodeType.PERSISTENT
)
print(f"创建持久节点: {create_result}")
# 创建临时节点
ephemeral_result = zk.create_node(
"/app/instance",
b"instance-1",
ZNodeType.EPHEMERAL,
session_id=session_id
)
print(f"创建临时节点: {ephemeral_result}")
# 创建顺序节点
sequential_result = zk.create_node(
"/app/worker-",
b"worker data",
ZNodeType.PERSISTENT_SEQUENTIAL
)
print(f"创建顺序节点: {sequential_result}")
# 获取节点数据
print("\n=== 获取节点数据 ===")
get_result = zk.get_node("/app", watch=True, session_id=session_id)
print(f"节点数据: {get_result}")
# 设置节点数据
print("\n=== 设置节点数据 ===")
set_result = zk.set_node_data("/app", b"updated config")
print(f"设置数据结果: {set_result}")
# 获取子节点
print("\n=== 获取子节点 ===")
children_result = zk.get_children("/app", watch=True, session_id=session_id)
print(f"子节点列表: {children_result}")
# 检查节点存在性
print("\n=== 检查节点存在性 ===")
exists_result = zk.exists("/app/nonexistent", watch=True, session_id=session_id)
print(f"节点存在性: {exists_result}")
# 获取集群状态
print("\n=== 集群状态 ===")
cluster_state = zk.get_cluster_state()
print(f"集群ID: {cluster_state['cluster_id']}")
print(f"Leader ID: {cluster_state['leader_id']}")
print(f"当前ZXID: {cluster_state['current_zxid']}")
print(f"服务器总数: {cluster_state['servers']['total']}")
print(f"节点总数: {cluster_state['nodes']['total']}")
print(f"活跃会话: {cluster_state['sessions']['active']}")
print(f"活跃监听器: {cluster_state['watchers']['active']}")
# 获取服务器信息
print("\n=== 服务器信息 ===")
if zk.leader_id:
server_info = zk.get_server_info(zk.leader_id)
print(f"Leader服务器: {server_info['hostname']}:{server_info['port']}")
print(f"服务器状态: {server_info['state']}")
print(f"Followers数量: {len(server_info['followers'])}")
# 获取会话信息
print("\n=== 会话信息 ===")
session_info = zk.get_session_info(session_id)
print(f"会话ID: {session_info['session_id']}")
print(f"客户端ID: {session_info['client_id']}")
print(f"会话状态: {session_info['state']}")
print(f"临时节点数: {len(session_info['ephemeral_nodes'])}")
print(f"监听器数: {session_info['watchers']}")
# 删除节点
print("\n=== 删除节点 ===")
delete_result = zk.delete_node("/app/instance", session_id=session_id)
print(f"删除临时节点: {delete_result}")
# 关闭会话
print("\n=== 关闭会话 ===")
close_result = zk.close_session(session_id)
print(f"会话关闭结果: {close_result}")
6. 监控管理组件
6.1 Prometheus监控详解
from typing import Dict, List, Any, Optional, Tuple, Union, Callable
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import re
import math
class MetricType(Enum):
"""指标类型"""
COUNTER = "计数器"
GAUGE = "仪表盘"
HISTOGRAM = "直方图"
SUMMARY = "摘要"
class AlertState(Enum):
"""告警状态"""
INACTIVE = "未激活"
PENDING = "待定"
FIRING = "触发"
RESOLVED = "已解决"
class ScrapeHealth(Enum):
"""抓取健康状态"""
UP = "正常"
DOWN = "异常"
UNKNOWN = "未知"
@dataclass
class MetricSample:
"""指标样本"""
timestamp: datetime
value: float
labels: Dict[str, str] = field(default_factory=dict)
@dataclass
class Metric:
"""Prometheus指标"""
name: str
metric_type: MetricType
help_text: str
samples: List[MetricSample] = field(default_factory=list)
labels: Dict[str, str] = field(default_factory=dict)
created_at: datetime = field(default_factory=datetime.now)
@dataclass
class ScrapeTarget:
"""抓取目标"""
job_name: str
instance: str
endpoint: str
labels: Dict[str, str]
scrape_interval: int # 秒
scrape_timeout: int # 秒
health: ScrapeHealth
last_scrape: Optional[datetime] = None
last_error: Optional[str] = None
metrics_count: int = 0
@dataclass
class AlertRule:
"""告警规则"""
name: str
expr: str # PromQL表达式
duration: int # 持续时间（秒）
severity: str
summary: str
description: str
labels: Dict[str, str] = field(default_factory=dict)
annotations: Dict[str, str] = field(default_factory=dict)
state: AlertState = AlertState.INACTIVE
active_at: Optional[datetime] = None
fired_at: Optional[datetime] = None
resolved_at: Optional[datetime] = None
value: Optional[float] = None
@dataclass
class Recording:
"""记录规则"""
name: str
expr: str # PromQL表达式
labels: Dict[str, str] = field(default_factory=dict)
interval: int = 60 # 评估间隔（秒）
last_evaluation: Optional[datetime] = None
last_value: Optional[float] = None
class PrometheusServer:
"""
Prometheus监控服务器
"""
def __init__(self, server_id: str = "prometheus-1"):
self.server_id = server_id
self.metrics = {} # metric_name -> Metric
self.targets = {} # target_id -> ScrapeTarget
self.alert_rules = {} # rule_name -> AlertRule
self.recordings = {} # recording_name -> Recording
self.active_alerts = {} # alert_id -> AlertRule
self.metric_lock = threading.Lock()
self.alert_lock = threading.Lock()
self.scrape_lock = threading.Lock()
# 配置参数
self.retention_period = timedelta(days=15) # 数据保留期
self.evaluation_interval = 15 # 规则评估间隔（秒）
self.scrape_interval = 15 # 默认抓取间隔（秒）
# 初始化默认指标和目标
self._initialize_default_metrics()
self._initialize_default_targets()
self._initialize_default_rules()
# 启动后台任务
self._start_background_tasks()
def _initialize_default_metrics(self):
"""初始化默认指标"""
default_metrics = [
{
'name': 'prometheus_build_info',
'type': MetricType.GAUGE,
'help': 'Prometheus build information',
'labels': {'version': '2.40.0', 'revision': 'abc123'}
},
{
'name': 'prometheus_config_last_reload_successful',
'type': MetricType.GAUGE,
'help': 'Whether the last configuration reload was successful'
},
{
'name': 'prometheus_tsdb_head_samples_appended_total',
'type': MetricType.COUNTER,
'help': 'Total number of appended samples'
},
{
'name': 'up',
'type': MetricType.GAUGE,
'help': 'Whether the instance is up'
}
]
for metric_def in default_metrics:
metric = Metric(
name=metric_def['name'],
metric_type=metric_def['type'],
help_text=metric_def['help'],
labels=metric_def.get('labels', {})
)
self.metrics[metric.name] = metric
def _initialize_default_targets(self):
"""初始化默认抓取目标"""
default_targets = [
{
'job_name': 'prometheus',
'instance': 'localhost:9090',
'endpoint': 'http://localhost:9090/metrics',
'labels': {'job': 'prometheus', 'instance': 'localhost:9090'}
},
{
'job_name': 'node-exporter',
'instance': 'localhost:9100',
'endpoint': 'http://localhost:9100/metrics',
'labels': {'job': 'node-exporter', 'instance': 'localhost:9100'}
},
{
'job_name': 'hadoop-namenode',
'instance': 'namenode:9870',
'endpoint': 'http://namenode:9870/jmx',
'labels': {'job': 'hadoop-namenode', 'instance': 'namenode:9870'}
}
]
for target_def in default_targets:
target_id = f"{target_def['job_name']}-{target_def['instance']}"
target = ScrapeTarget(
job_name=target_def['job_name'],
instance=target_def['instance'],
endpoint=target_def['endpoint'],
labels=target_def['labels'],
scrape_interval=self.scrape_interval,
scrape_timeout=10,
health=ScrapeHealth.UNKNOWN
)
self.targets[target_id] = target
def _initialize_default_rules(self):
"""初始化默认告警规则"""
default_rules = [
{
'name': 'InstanceDown',
'expr': 'up == 0',
'duration': 300, # 5分钟
'severity': 'critical',
'summary': 'Instance {{ $labels.instance }} down',
'description': 'Instance {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.'
},
{
'name': 'HighCPUUsage',
'expr': '100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80',
'duration': 300,
'severity': 'warning',
'summary': 'High CPU usage on {{ $labels.instance }}',
'description': 'CPU usage is above 80% on {{ $labels.instance }} for more than 5 minutes.'
},
{
'name': 'HighMemoryUsage',
'expr': '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90',
'duration': 300,
'severity': 'warning',
'summary': 'High memory usage on {{ $labels.instance }}',
'description': 'Memory usage is above 90% on {{ $labels.instance }} for more than 5 minutes.'
}
]
for rule_def in default_rules:
rule = AlertRule(
name=rule_def['name'],
expr=rule_def['expr'],
duration=rule_def['duration'],
severity=rule_def['severity'],
summary=rule_def['summary'],
description=rule_def['description']
)
self.alert_rules[rule.name] = rule
def _start_background_tasks(self):
"""启动后台任务"""
# 这里可以启动实际的后台线程
# 为了演示，我们只是标记任务已启动
pass
def add_target(self, job_name: str, instance: str, endpoint: str,
labels: Optional[Dict[str, str]] = None,
scrape_interval: int = None) -> Dict[str, Any]:
"""
添加抓取目标
Args:
job_name: 作业名称
instance: 实例标识
endpoint: 抓取端点
labels: 标签
scrape_interval: 抓取间隔（秒）
Returns:
Dict[str, Any]: 添加结果
"""
target_id = f"{job_name}-{instance}"
if target_id in self.targets:
return {'status': 'error', 'message': f'Target {target_id} already exists'}
if labels is None:
labels = {'job': job_name, 'instance': instance}
target = ScrapeTarget(
job_name=job_name,
instance=instance,
endpoint=endpoint,
labels=labels,
scrape_interval=scrape_interval or self.scrape_interval,
scrape_timeout=10,
health=ScrapeHealth.UNKNOWN
)
with self.scrape_lock:
self.targets[target_id] = target
return {
'status': 'success',
'target_id': target_id,
'job_name': job_name,
'instance': instance,
'endpoint': endpoint
}
def remove_target(self, target_id: str) -> Dict[str, Any]:
"""
移除抓取目标
Args:
target_id: 目标ID
Returns:
Dict[str, Any]: 移除结果
"""
if target_id not in self.targets:
return {'status': 'error', 'message': f'Target {target_id} not found'}
with self.scrape_lock:
target = self.targets.pop(target_id)
return {
'status': 'success',
'target_id': target_id,
'job_name': target.job_name,
'instance': target.instance
}
def scrape_target(self, target_id: str) -> Dict[str, Any]:
"""
抓取目标指标
Args:
target_id: 目标ID
Returns:
Dict[str, Any]: 抓取结果
"""
if target_id not in self.targets:
return {'status': 'error', 'message': f'Target {target_id} not found'}
target = self.targets[target_id]
try:
# 模拟抓取过程
scraped_metrics = self._simulate_scrape(target)
# 更新目标状态
target.health = ScrapeHealth.UP
target.last_scrape = datetime.now()
target.last_error = None
target.metrics_count = len(scraped_metrics)
# 存储指标
with self.metric_lock:
for metric_name, samples in scraped_metrics.items():
if metric_name not in self.metrics:
# 创建新指标
self.metrics[metric_name] = Metric(
name=metric_name,
metric_type=MetricType.GAUGE, # 默认类型
help_text=f"Metric {metric_name} from {target.job_name}"
)
# 添加样本
metric = self.metrics[metric_name]
for sample in samples:
# 合并目标标签和样本标签
combined_labels = {**target.labels, **sample.labels}
sample.labels = combined_labels
metric.samples.append(sample)
# 保持样本数量在合理范围内
if len(metric.samples) > 1000:
metric.samples = metric.samples[-1000:]
return {
'status': 'success',
'target_id': target_id,
'metrics_scraped': len(scraped_metrics),
'scrape_duration': random.uniform(0.1, 2.0),
'timestamp': target.last_scrape.isoformat()
}
except Exception as e:
# 更新目标状态为异常
target.health = ScrapeHealth.DOWN
target.last_scrape = datetime.now()
target.last_error = str(e)
return {
'status': 'error',
'target_id': target_id,
'error': str(e),
'timestamp': target.last_scrape.isoformat()
}
def _simulate_scrape(self, target: ScrapeTarget) -> Dict[str, List[MetricSample]]:
"""
模拟抓取指标数据
Args:
target: 抓取目标
Returns:
Dict[str, List[MetricSample]]: 抓取的指标数据
"""
now = datetime.now()
metrics = {}
if target.job_name == 'prometheus':
# Prometheus自身指标
metrics['prometheus_config_last_reload_successful'] = [
MetricSample(timestamp=now, value=1.0)
]
metrics['prometheus_tsdb_head_samples_appended_total'] = [
MetricSample(timestamp=now, value=random.randint(10000, 50000))
]
elif target.job_name == 'node-exporter':
# 节点导出器指标
metrics['node_cpu_seconds_total'] = [
MetricSample(timestamp=now, value=random.uniform(1000, 5000),
labels={'mode': 'idle', 'cpu': '0'}),
MetricSample(timestamp=now, value=random.uniform(100, 500),
labels={'mode': 'user', 'cpu': '0'}),
MetricSample(timestamp=now, value=random.uniform(50, 200),
labels={'mode': 'system', 'cpu': '0'})
]
metrics['node_memory_MemTotal_bytes'] = [
MetricSample(timestamp=now, value=8589934592) # 8GB
]
metrics['node_memory_MemAvailable_bytes'] = [
MetricSample(timestamp=now, value=random.randint(2000000000, 6000000000))
]
elif target.job_name == 'hadoop-namenode':
# Hadoop NameNode指标
metrics['hadoop_namenode_capacity_total'] = [
MetricSample(timestamp=now, value=1099511627776) # 1TB
]
metrics['hadoop_namenode_capacity_used'] = [
MetricSample(timestamp=now, value=random.randint(100000000000, 500000000000))
]
metrics['hadoop_namenode_blocks_total'] = [
MetricSample(timestamp=now, value=random.randint(10000, 100000))
]
# 添加通用的up指标
metrics['up'] = [MetricSample(timestamp=now, value=1.0)]
return metrics
def query(self, query_expr: str, timestamp: Optional[datetime] = None) -> Dict[str, Any]:
"""
执行PromQL查询
Args:
query_expr: PromQL查询表达式
timestamp: 查询时间点
Returns:
Dict[str, Any]: 查询结果
"""
if timestamp is None:
timestamp = datetime.now()
try:
# 简化的PromQL解析和执行
result = self._execute_promql(query_expr, timestamp)
return {
'status': 'success',
'query': query_expr,
'timestamp': timestamp.isoformat(),
'result_type': 'vector',
'result': result
}
except Exception as e:
return {
'status': 'error',
'query': query_expr,
'error': str(e)
}
def _execute_promql(self, query_expr: str, timestamp: datetime) -> List[Dict[str, Any]]:
"""
执行PromQL表达式（简化版本）
Args:
query_expr: PromQL表达式
timestamp: 查询时间
Returns:
List[Dict[str, Any]]: 查询结果
"""
results = []
# 简单的指标名匹配
if query_expr in self.metrics:
metric = self.metrics[query_expr]
# 返回最新的样本
latest_samples = {}
for sample in metric.samples:
label_key = json.dumps(sample.labels, sort_keys=True)
if label_key not in latest_samples or sample.timestamp > latest_samples[label_key].timestamp:
latest_samples[label_key] = sample
for sample in latest_samples.values():
results.append({
'metric': sample.labels,
'value': [timestamp.timestamp(), str(sample.value)]
})
# 处理简单的比较表达式
elif ' == ' in query_expr or ' > ' in query_expr or ' < ' in query_expr:
# 例如: up == 0
parts = re.split(r'\s*(==|>|<|>=|<=|!=)\s*', query_expr)
if len(parts) >= 3:
metric_name = parts[0].strip()
operator = parts[1].strip()
threshold = float(parts[2].strip())
if metric_name in self.metrics:
metric = self.metrics[metric_name]
for sample in metric.samples:
if self._compare_value(sample.value, operator, threshold):
results.append({
'metric': sample.labels,
'value': [timestamp.timestamp(), str(sample.value)]
})
return results
def _compare_value(self, value: float, operator: str, threshold: float) -> bool:
"""
比较值
Args:
value: 实际值
operator: 比较操作符
threshold: 阈值
Returns:
bool: 比较结果
"""
if operator == '==':
return value == threshold
elif operator == '!=':
return value != threshold
elif operator == '>':
return value > threshold
elif operator == '<':
return value < threshold
elif operator == '>=':
return value >= threshold
elif operator == '<=':
return value <= threshold
return False
def add_alert_rule(self, name: str, expr: str, duration: int, severity: str,
summary: str, description: str,
labels: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
"""
添加告警规则
Args:
name: 规则名称
expr: PromQL表达式
duration: 持续时间（秒）
severity: 严重程度
summary: 摘要
description: 描述
labels: 标签
Returns:
Dict[str, Any]: 添加结果
"""
if name in self.alert_rules:
return {'status': 'error', 'message': f'Alert rule {name} already exists'}
rule = AlertRule(
name=name,
expr=expr,
duration=duration,
severity=severity,
summary=summary,
description=description,
labels=labels or {}
)
with self.alert_lock:
self.alert_rules[name] = rule
return {
'status': 'success',
'rule_name': name,
'expr': expr,
'severity': severity
}
def evaluate_alerts(self) -> Dict[str, Any]:
"""
评估告警规则
Returns:
Dict[str, Any]: 评估结果
"""
now = datetime.now()
evaluated_rules = 0
fired_alerts = 0
resolved_alerts = 0
with self.alert_lock:
for rule_name, rule in self.alert_rules.items():
try:
# 执行PromQL查询
query_result = self.query(rule.expr, now)
if query_result['status'] == 'success' and query_result['result']:
# 有匹配的结果，告警条件满足
if rule.state == AlertState.INACTIVE:
rule.state = AlertState.PENDING
rule.active_at = now
elif rule.state == AlertState.PENDING:
# 检查是否超过持续时间
if (now - rule.active_at).total_seconds() >= rule.duration:
rule.state = AlertState.FIRING
rule.fired_at = now
self.active_alerts[f"{rule_name}-{uuid.uuid4().hex[:8]}"] = rule
fired_alerts += 1
else:
# 告警条件不满足
if rule.state in [AlertState.PENDING, AlertState.FIRING]:
rule.state = AlertState.RESOLVED
rule.resolved_at = now
resolved_alerts += 1
# 从活跃告警中移除
to_remove = [alert_id for alert_id, alert in self.active_alerts.items()
if alert.name == rule_name]
for alert_id in to_remove:
del self.active_alerts[alert_id]
evaluated_rules += 1
except Exception as e:
# 规则评估失败
continue
return {
'status': 'success',
'timestamp': now.isoformat(),
'evaluated_rules': evaluated_rules,
'fired_alerts': fired_alerts,
'resolved_alerts': resolved_alerts,
'active_alerts': len(self.active_alerts)
}
def get_targets(self) -> Dict[str, Any]:
"""
获取所有抓取目标
Returns:
Dict[str, Any]: 目标列表
"""
targets_info = []
for target_id, target in self.targets.items():
targets_info.append({
'target_id': target_id,
'job_name': target.job_name,
'instance': target.instance,
'endpoint': target.endpoint,
'health': target.health.value,
'last_scrape': target.last_scrape.isoformat() if target.last_scrape else None,
'last_error': target.last_error,
'metrics_count': target.metrics_count,
'scrape_interval': target.scrape_interval,
'labels': target.labels
})
return {
'status': 'success',
'targets': targets_info,
'total_targets': len(targets_info)
}
def get_alerts(self) -> Dict[str, Any]:
"""
获取告警信息
Returns:
Dict[str, Any]: 告警信息
"""
alerts_info = []
for alert_id, alert in self.active_alerts.items():
alerts_info.append({
'alert_id': alert_id,
'name': alert.name,
'state': alert.state.value,
'severity': alert.severity,
'summary': alert.summary,
'description': alert.description,
'active_at': alert.active_at.isoformat() if alert.active_at else None,
'fired_at': alert.fired_at.isoformat() if alert.fired_at else None,
'labels': alert.labels,
'annotations': alert.annotations
})
rules_info = []
for rule_name, rule in self.alert_rules.items():
rules_info.append({
'name': rule.name,
'expr': rule.expr,
'duration': rule.duration,
'severity': rule.severity,
'state': rule.state.value,
'summary': rule.summary
})
return {
'status': 'success',
'active_alerts': alerts_info,
'alert_rules': rules_info,
'total_active_alerts': len(alerts_info),
'total_rules': len(rules_info)
}
def get_metrics(self, metric_name: Optional[str] = None) -> Dict[str, Any]:
"""
获取指标信息
Args:
metric_name: 指标名称（可选）
Returns:
Dict[str, Any]: 指标信息
"""
if metric_name:
if metric_name not in self.metrics:
return {'status': 'error', 'message': f'Metric {metric_name} not found'}
metric = self.metrics[metric_name]
return {
'status': 'success',
'metric': {
'name': metric.name,
'type': metric.metric_type.value,
'help': metric.help_text,
'samples_count': len(metric.samples),
'labels': metric.labels,
'created_at': metric.created_at.isoformat()
}
}
else:
metrics_info = []
for metric_name, metric in self.metrics.items():
metrics_info.append({
'name': metric.name,
'type': metric.metric_type.value,
'help': metric.help_text,
'samples_count': len(metric.samples),
'labels': metric.labels
})
return {
'status': 'success',
'metrics': metrics_info,
'total_metrics': len(metrics_info)
}
def get_server_status(self) -> Dict[str, Any]:
"""
获取服务器状态
Returns:
Dict[str, Any]: 服务器状态
"""
# 统计指标样本总数
total_samples = sum(len(metric.samples) for metric in self.metrics.values())
# 统计健康目标数
healthy_targets = sum(1 for target in self.targets.values()
if target.health == ScrapeHealth.UP)
return {
'server_id': self.server_id,
'status': 'running',
'uptime': '24h 30m 15s', # 模拟运行时间
'version': '2.40.0',
'build_info': {
'version': '2.40.0',
'revision': 'abc123def',
'branch': 'HEAD',
'build_date': '2023-12-01T10:00:00Z'
},
'config': {
'retention_period': str(self.retention_period),
'evaluation_interval': f'{self.evaluation_interval}s',
'scrape_interval': f'{self.scrape_interval}s'
},
'stats': {
'total_metrics': len(self.metrics),
'total_samples': total_samples,
'total_targets': len(self.targets),
'healthy_targets': healthy_targets,
'total_alert_rules': len(self.alert_rules),
'active_alerts': len(self.active_alerts)
},
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Prometheus服务器
prometheus = PrometheusServer("prod-prometheus")
print("=== Prometheus监控服务器示例 ===")
# 添加抓取目标
print("\n=== 添加抓取目标 ===")
target_result = prometheus.add_target(
job_name="webapp",
instance="webapp-1:8080",
endpoint="http://webapp-1:8080/metrics",
labels={'environment': 'production', 'service': 'webapp'},
scrape_interval=30
)
print(f"添加目标结果: {target_result}")
# 抓取目标指标
print("\n=== 抓取指标 ===")
scrape_result = prometheus.scrape_target("webapp-webapp-1:8080")
print(f"抓取结果: {scrape_result}")
# 执行查询
print("\n=== 执行PromQL查询 ===")
query_result = prometheus.query("up")
print(f"查询结果: {query_result}")
# 添加告警规则
print("\n=== 添加告警规则 ===")
alert_result = prometheus.add_alert_rule(
name="WebAppDown",
expr="up{job='webapp'} == 0",
duration=300,
severity="critical",
summary="WebApp instance down",
description="WebApp instance {{ $labels.instance }} has been down for more than 5 minutes."
)
print(f"添加告警规则: {alert_result}")
# 评估告警
print("\n=== 评估告警 ===")
eval_result = prometheus.evaluate_alerts()
print(f"告警评估结果: {eval_result}")
# 获取目标信息
print("\n=== 抓取目标信息 ===")
targets_info = prometheus.get_targets()
print(f"目标总数: {targets_info['total_targets']}")
for target in targets_info['targets'][:2]: # 显示前2个目标
print(f" - {target['job_name']}/{target['instance']}: {target['health']}")
# 获取告警信息
print("\n=== 告警信息 ===")
alerts_info = prometheus.get_alerts()
print(f"活跃告警数: {alerts_info['total_active_alerts']}")
print(f"告警规则数: {alerts_info['total_rules']}")
# 获取指标信息
print("\n=== 指标信息 ===")
metrics_info = prometheus.get_metrics()
print(f"指标总数: {metrics_info['total_metrics']}")
for metric in metrics_info['metrics'][:3]: # 显示前3个指标
print(f" - {metric['name']} ({metric['type']}): {metric['samples_count']} samples")
# 获取服务器状态
print("\n=== 服务器状态 ===")
server_status = prometheus.get_server_status()
print(f"服务器ID: {server_status['server_id']}")
print(f"状态: {server_status['status']}")
print(f"版本: {server_status['version']}")
print(f"运行时间: {server_status['uptime']}")
print(f"指标数: {server_status['stats']['total_metrics']}")
print(f"样本数: {server_status['stats']['total_samples']}")
print(f"健康目标: {server_status['stats']['healthy_targets']}/{server_status['stats']['total_targets']}")
6.2 Grafana可视化详解
from typing import Dict, List, Any, Optional, Tuple, Union, Callable
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import copy
class DashboardStatus(Enum):
"""仪表板状态"""
ACTIVE = "活跃"
INACTIVE = "非活跃"
ARCHIVED = "已归档"
DRAFT = "草稿"
class PanelType(Enum):
"""面板类型"""
GRAPH = "图表"
SINGLESTAT = "单一统计"
TABLE = "表格"
HEATMAP = "热力图"
GAUGE = "仪表盘"
BAR_GAUGE = "条形仪表盘"
STAT = "统计"
TEXT = "文本"
LOGS = "日志"
ALERT_LIST = "告警列表"
class DataSourceType(Enum):
"""数据源类型"""
PROMETHEUS = "Prometheus"
INFLUXDB = "InfluxDB"
ELASTICSEARCH = "Elasticsearch"
MYSQL = "MySQL"
POSTGRES = "PostgreSQL"
GRAPHITE = "Graphite"
CLOUDWATCH = "CloudWatch"
LOKI = "Loki"
class AlertState(Enum):
"""告警状态"""
NO_DATA = "无数据"
PAUSED = "暂停"
ALERTING = "告警中"
OK = "正常"
PENDING = "待定"
class UserRole(Enum):
"""用户角色"""
VIEWER = "查看者"
EDITOR = "编辑者"
ADMIN = "管理员"
@dataclass
class DataSource:
"""数据源"""
id: str
name: str
type: DataSourceType
url: str
access: str = "proxy" # proxy, direct
database: str = ""
user: str = ""
password: str = ""
basic_auth: bool = False
basic_auth_user: str = ""
basic_auth_password: str = ""
with_credentials: bool = False
is_default: bool = False
json_data: Dict[str, Any] = field(default_factory=dict)
secure_json_data: Dict[str, Any] = field(default_factory=dict)
version: int = 1
read_only: bool = False
created_at: datetime = field(default_factory=datetime.now)
updated_at: datetime = field(default_factory=datetime.now)
@dataclass
class Target:
"""查询目标"""
expr: str # 查询表达式
interval: str = "" # 查询间隔
legend_format: str = "" # 图例格式
ref_id: str = "A" # 引用ID
hide: bool = False
format: str = "time_series" # time_series, table, heatmap
instant: bool = False
interval_factor: int = 1
max_data_points: int = 300
step: int = 60
@dataclass
class GridPos:
"""网格位置"""
h: int # 高度
w: int # 宽度
x: int # X坐标
y: int # Y坐标
@dataclass
class Panel:
"""面板"""
id: int
title: str
type: PanelType
grid_pos: GridPos
targets: List[Target] = field(default_factory=list)
datasource: Optional[str] = None
description: str = ""
transparent: bool = False
options: Dict[str, Any] = field(default_factory=dict)
field_config: Dict[str, Any] = field(default_factory=dict)
alert: Optional[Dict[str, Any]] = None
links: List[Dict[str, Any]] = field(default_factory=list)
repeat: Optional[str] = None
repeat_direction: str = "h" # h, v
max_per_row: Optional[int] = None
collapsed: bool = False
panels: List['Panel'] = field(default_factory=list) # 用于行面板
@dataclass
class TemplateVariable:
"""模板变量"""
name: str
type: str # query, custom, constant, datasource, interval, textbox, adhoc
label: str = ""
description: str = ""
query: str = ""
datasource: Optional[str] = None
refresh: str = "never" # never, on_dashboard_load, on_time_range_change
regex: str = ""
sort: int = 0 # 0: disabled, 1: alphabetical asc, 2: alphabetical desc, 3: numerical asc, 4: numerical desc
multi: bool = False
include_all: bool = False
all_value: Optional[str] = None
options: List[Dict[str, Any]] = field(default_factory=list)
current: Dict[str, Any] = field(default_factory=dict)
hide: int = 0 # 0: visible, 1: hidden, 2: variable
@dataclass
class TimeRange:
"""时间范围"""
from_time: str = "now-6h"
to_time: str = "now"
@dataclass
class Dashboard:
"""仪表板"""
id: Optional[int] = None
uid: str = field(default_factory=lambda: str(uuid.uuid4()))
title: str = "New Dashboard"
description: str = ""
tags: List[str] = field(default_factory=list)
style: str = "dark" # dark, light
timezone: str = "browser"
panels: List[Panel] = field(default_factory=list)
templating: List[TemplateVariable] = field(default_factory=list)
time: TimeRange = field(default_factory=TimeRange)
time_picker: Dict[str, Any] = field(default_factory=dict)
refresh: str = "5s"
schema_version: int = 30
version: int = 1
links: List[Dict[str, Any]] = field(default_factory=list)
annotations: Dict[str, Any] = field(default_factory=dict)
editable: bool = True
graph_tooltip: str = "shared_crosshair" # default, shared_crosshair, shared_tooltip
hide_controls: bool = False
status: DashboardStatus = DashboardStatus.ACTIVE
folder_id: Optional[int] = None
folder_title: str = "General"
created_at: datetime = field(default_factory=datetime.now)
updated_at: datetime = field(default_factory=datetime.now)
created_by: str = "admin"
updated_by: str = "admin"
@dataclass
class User:
"""用户"""
id: int
login: str
name: str
email: str
role: UserRole
is_admin: bool = False
is_disabled: bool = False
last_seen_at: Optional[datetime] = None
created_at: datetime = field(default_factory=datetime.now)
updated_at: datetime = field(default_factory=datetime.now)
avatar_url: str = ""
org_id: int = 1
theme: str = "" # light, dark, ""
timezone: str = ""
help_flags1: int = 0
email_verified: bool = False
@dataclass
class Organization:
"""组织"""
id: int
name: str
address: Dict[str, str] = field(default_factory=dict)
created_at: datetime = field(default_factory=datetime.now)
updated_at: datetime = field(default_factory=datetime.now)
@dataclass
class Folder:
"""文件夹"""
id: int
uid: str
title: str
url: str
has_acl: bool = False
can_save: bool = True
can_edit: bool = True
can_admin: bool = True
created_at: datetime = field(default_factory=datetime.now)
updated_at: datetime = field(default_factory=datetime.now)
created_by: str = "admin"
updated_by: str = "admin"
version: int = 1
class GrafanaServer:
"""
Grafana可视化服务器
"""
def __init__(self, server_id: str = "grafana-1"):
self.server_id = server_id
self.datasources = {} # datasource_id -> DataSource
self.dashboards = {} # dashboard_uid -> Dashboard
self.users = {} # user_id -> User
self.organizations = {} # org_id -> Organization
self.folders = {} # folder_id -> Folder
self.sessions = {} # session_id -> user_id
# 线程锁
self.datasource_lock = threading.Lock()
self.dashboard_lock = threading.Lock()
self.user_lock = threading.Lock()
# 配置参数
self.version = "9.3.0"
self.build_info = {
'version': '9.3.0',
'commit': 'abc123def',
'build_date': '2023-12-01T10:00:00Z',
'edition': 'Open Source'
}
# 初始化默认数据
self._initialize_default_data()
def _initialize_default_data(self):
"""初始化默认数据"""
# 创建默认组织
default_org = Organization(
id=1,
name="Main Org."
)
self.organizations[1] = default_org
# 创建默认用户
admin_user = User(
id=1,
login="admin",
name="Admin",
email="admin@localhost",
role=UserRole.ADMIN,
is_admin=True,
org_id=1
)
self.users[1] = admin_user
# 创建默认文件夹
general_folder = Folder(
id=0,
uid="general",
title="General",
url="/dashboards/f/general/general"
)
self.folders[0] = general_folder
# 创建默认数据源
prometheus_ds = DataSource(
id="prometheus-1",
name="Prometheus",
type=DataSourceType.PROMETHEUS,
url="http://localhost:9090",
access="proxy",
is_default=True,
json_data={
'httpMethod': 'POST',
'queryTimeout': '60s',
'timeInterval': '15s'
}
)
self.datasources["prometheus-1"] = prometheus_ds
# 创建默认仪表板
self._create_default_dashboards()
def _create_default_dashboards(self):
"""创建默认仪表板"""
# 系统监控仪表板
system_dashboard = Dashboard(
uid="system-overview",
title="System Overview",
description="System monitoring dashboard",
tags=["system", "monitoring"],
folder_id=0
)
# 添加CPU使用率面板
cpu_panel = Panel(
id=1,
title="CPU Usage",
type=PanelType.GRAPH,
grid_pos=GridPos(h=8, w=12, x=0, y=0),
targets=[
Target(
expr='100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
legend_format="{{instance}}",
ref_id="A"
)
],
datasource="prometheus-1"
)
# 添加内存使用率面板
memory_panel = Panel(
id=2,
title="Memory Usage",
type=PanelType.GRAPH,
grid_pos=GridPos(h=8, w=12, x=12, y=0),
targets=[
Target(
expr='(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
legend_format="{{instance}}",
ref_id="A"
)
],
datasource="prometheus-1"
)
# 添加磁盘使用率面板
disk_panel = Panel(
id=3,
title="Disk Usage",
type=PanelType.GAUGE,
grid_pos=GridPos(h=8, w=12, x=0, y=8),
targets=[
Target(
expr='(1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) * 100',
legend_format="{{instance}} - {{mountpoint}}",
ref_id="A"
)
],
datasource="prometheus-1"
)
# 添加网络流量面板
network_panel = Panel(
id=4,
title="Network Traffic",
type=PanelType.GRAPH,
grid_pos=GridPos(h=8, w=12, x=12, y=8),
targets=[
Target(
expr='rate(node_network_receive_bytes_total[5m])',
legend_format="{{instance}} - {{device}} (in)",
ref_id="A"
),
Target(
expr='rate(node_network_transmit_bytes_total[5m])',
legend_format="{{instance}} - {{device}} (out)",
ref_id="B"
)
],
datasource="prometheus-1"
)
system_dashboard.panels = [cpu_panel, memory_panel, disk_panel, network_panel]
self.dashboards[system_dashboard.uid] = system_dashboard
# Hadoop监控仪表板
hadoop_dashboard = Dashboard(
uid="hadoop-overview",
title="Hadoop Cluster Overview",
description="Hadoop cluster monitoring dashboard",
tags=["hadoop", "hdfs", "yarn"],
folder_id=0
)
# HDFS容量面板
hdfs_capacity_panel = Panel(
id=1,
title="HDFS Capacity",
type=PanelType.STAT,
grid_pos=GridPos(h=6, w=8, x=0, y=0),
targets=[
Target(
expr='hadoop_namenode_capacity_used / hadoop_namenode_capacity_total * 100',
legend_format="Used %",
ref_id="A"
)
],
datasource="prometheus-1"
)
# HDFS块数量面板
hdfs_blocks_panel = Panel(
id=2,
title="HDFS Blocks",
type=PanelType.SINGLESTAT,
grid_pos=GridPos(h=6, w=8, x=8, y=0),
targets=[
Target(
expr='hadoop_namenode_blocks_total',
legend_format="Total Blocks",
ref_id="A"
)
],
datasource="prometheus-1"
)
# DataNode状态面板
datanode_status_panel = Panel(
id=3,
title="DataNode Status",
type=PanelType.TABLE,
grid_pos=GridPos(h=6, w=8, x=16, y=0),
targets=[
Target(
expr='up{job="hadoop-datanode"}',
legend_format="{{instance}}",
ref_id="A",
format="table"
)
],
datasource="prometheus-1"
)
hadoop_dashboard.panels = [hdfs_capacity_panel, hdfs_blocks_panel, datanode_status_panel]
self.dashboards[hadoop_dashboard.uid] = hadoop_dashboard
def add_datasource(self, name: str, ds_type: DataSourceType, url: str,
access: str = "proxy", database: str = "",
user: str = "", password: str = "",
is_default: bool = False,
json_data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
添加数据源
Args:
name: 数据源名称
ds_type: 数据源类型
url: 数据源URL
access: 访问方式
database: 数据库名称
user: 用户名
password: 密码
is_default: 是否为默认数据源
json_data: JSON配置数据
Returns:
Dict[str, Any]: 添加结果
"""
ds_id = f"{ds_type.value.lower()}-{len(self.datasources) + 1}"
# 检查名称是否已存在
for ds in self.datasources.values():
if ds.name == name:
return {'status': 'error', 'message': f'Datasource with name {name} already exists'}
# 如果设置为默认数据源，取消其他数据源的默认状态
if is_default:
with self.datasource_lock:
for ds in self.datasources.values():
ds.is_default = False
datasource = DataSource(
id=ds_id,
name=name,
type=ds_type,
url=url,
access=access,
database=database,
user=user,
password=password,
is_default=is_default,
json_data=json_data or {}
)
with self.datasource_lock:
self.datasources[ds_id] = datasource
return {
'status': 'success',
'datasource': {
'id': ds_id,
'name': name,
'type': ds_type.value,
'url': url,
'is_default': is_default
}
}
def get_datasources(self) -> Dict[str, Any]:
"""
获取所有数据源
Returns:
Dict[str, Any]: 数据源列表
"""
datasources_info = []
for ds_id, ds in self.datasources.items():
datasources_info.append({
'id': ds.id,
'name': ds.name,
'type': ds.type.value,
'url': ds.url,
'access': ds.access,
'database': ds.database,
'is_default': ds.is_default,
'read_only': ds.read_only,
'created_at': ds.created_at.isoformat(),
'updated_at': ds.updated_at.isoformat()
})
return {
'status': 'success',
'datasources': datasources_info,
'total': len(datasources_info)
}
def create_dashboard(self, title: str, description: str = "",
tags: Optional[List[str]] = None,
folder_id: int = 0) -> Dict[str, Any]:
"""
创建仪表板
Args:
title: 仪表板标题
description: 描述
tags: 标签
folder_id: 文件夹ID
Returns:
Dict[str, Any]: 创建结果
"""
dashboard = Dashboard(
title=title,
description=description,
tags=tags or [],
folder_id=folder_id
)
with self.dashboard_lock:
self.dashboards[dashboard.uid] = dashboard
return {
'status': 'success',
'dashboard': {
'id': dashboard.id,
'uid': dashboard.uid,
'title': dashboard.title,
'url': f'/d/{dashboard.uid}/{dashboard.title.lower().replace(" ", "-")}',
'status': dashboard.status.value,
'version': dashboard.version
}
}
def get_dashboard(self, uid: str) -> Dict[str, Any]:
"""
获取仪表板
Args:
uid: 仪表板UID
Returns:
Dict[str, Any]: 仪表板信息
"""
if uid not in self.dashboards:
return {'status': 'error', 'message': f'Dashboard {uid} not found'}
dashboard = self.dashboards[uid]
# 转换面板信息
panels_info = []
for panel in dashboard.panels:
targets_info = []
for target in panel.targets:
targets_info.append({
'expr': target.expr,
'interval': target.interval,
'legend_format': target.legend_format,
'ref_id': target.ref_id,
'hide': target.hide
})
panels_info.append({
'id': panel.id,
'title': panel.title,
'type': panel.type.value,
'grid_pos': {
'h': panel.grid_pos.h,
'w': panel.grid_pos.w,
'x': panel.grid_pos.x,
'y': panel.grid_pos.y
},
'targets': targets_info,
'datasource': panel.datasource,
'description': panel.description
})
return {
'status': 'success',
'dashboard': {
'id': dashboard.id,
'uid': dashboard.uid,
'title': dashboard.title,
'description': dashboard.description,
'tags': dashboard.tags,
'style': dashboard.style,
'timezone': dashboard.timezone,
'panels': panels_info,
'time': {
'from': dashboard.time.from_time,
'to': dashboard.time.to_time
},
'refresh': dashboard.refresh,
'version': dashboard.version,
'editable': dashboard.editable,
'folder_id': dashboard.folder_id,
'folder_title': dashboard.folder_title,
'created_at': dashboard.created_at.isoformat(),
'updated_at': dashboard.updated_at.isoformat()
}
}
def add_panel_to_dashboard(self, dashboard_uid: str, panel_title: str,
panel_type: PanelType, grid_pos: Dict[str, int],
targets: List[Dict[str, Any]],
datasource: str = None) -> Dict[str, Any]:
"""
向仪表板添加面板
Args:
dashboard_uid: 仪表板UID
panel_title: 面板标题
panel_type: 面板类型
grid_pos: 网格位置
targets: 查询目标列表
datasource: 数据源
Returns:
Dict[str, Any]: 添加结果
"""
if dashboard_uid not in self.dashboards:
return {'status': 'error', 'message': f'Dashboard {dashboard_uid} not found'}
dashboard = self.dashboards[dashboard_uid]
# 生成新的面板ID
panel_id = max([p.id for p in dashboard.panels], default=0) + 1
# 创建查询目标
panel_targets = []
for target_data in targets:
target = Target(
expr=target_data.get('expr', ''),
interval=target_data.get('interval', ''),
legend_format=target_data.get('legend_format', ''),
ref_id=target_data.get('ref_id', 'A'),
hide=target_data.get('hide', False)
)
panel_targets.append(target)
# 创建面板
panel = Panel(
id=panel_id,
title=panel_title,
type=panel_type,
grid_pos=GridPos(
h=grid_pos.get('h', 8),
w=grid_pos.get('w', 12),
x=grid_pos.get('x', 0),
y=grid_pos.get('y', 0)
),
targets=panel_targets,
datasource=datasource
)
with self.dashboard_lock:
dashboard.panels.append(panel)
dashboard.version += 1
dashboard.updated_at = datetime.now()
return {
'status': 'success',
'panel': {
'id': panel_id,
'title': panel_title,
'type': panel_type.value,
'dashboard_uid': dashboard_uid
}
}
def get_dashboards(self, folder_id: Optional[int] = None,
tag: Optional[str] = None) -> Dict[str, Any]:
"""
获取仪表板列表
Args:
folder_id: 文件夹ID过滤
tag: 标签过滤
Returns:
Dict[str, Any]: 仪表板列表
"""
dashboards_info = []
for uid, dashboard in self.dashboards.items():
# 应用过滤条件
if folder_id is not None and dashboard.folder_id != folder_id:
continue
if tag is not None and tag not in dashboard.tags:
continue
dashboards_info.append({
'id': dashboard.id,
'uid': dashboard.uid,
'title': dashboard.title,
'description': dashboard.description,
'tags': dashboard.tags,
'folder_id': dashboard.folder_id,
'folder_title': dashboard.folder_title,
'status': dashboard.status.value,
'url': f'/d/{dashboard.uid}/{dashboard.title.lower().replace(" ", "-")}',
'version': dashboard.version,
'created_at': dashboard.created_at.isoformat(),
'updated_at': dashboard.updated_at.isoformat(),
'created_by': dashboard.created_by,
'updated_by': dashboard.updated_by
})
return {
'status': 'success',
'dashboards': dashboards_info,
'total': len(dashboards_info)
}
def delete_dashboard(self, uid: str) -> Dict[str, Any]:
"""
删除仪表板
Args:
uid: 仪表板UID
Returns:
Dict[str, Any]: 删除结果
"""
if uid not in self.dashboards:
return {'status': 'error', 'message': f'Dashboard {uid} not found'}
with self.dashboard_lock:
dashboard = self.dashboards.pop(uid)
return {
'status': 'success',
'message': f'Dashboard {dashboard.title} deleted successfully',
'dashboard_uid': uid
}
def create_user(self, login: str, name: str, email: str,
role: UserRole, password: str = "admin") -> Dict[str, Any]:
"""
创建用户
Args:
login: 登录名
name: 用户名
email: 邮箱
role: 角色
password: 密码
Returns:
Dict[str, Any]: 创建结果
"""
# 检查登录名是否已存在
for user in self.users.values():
if user.login == login:
return {'status': 'error', 'message': f'User with login {login} already exists'}
user_id = max(self.users.keys(), default=0) + 1
user = User(
id=user_id,
login=login,
name=name,
email=email,
role=role,
is_admin=(role == UserRole.ADMIN)
)
with self.user_lock:
self.users[user_id] = user
return {
'status': 'success',
'user': {
'id': user_id,
'login': login,
'name': name,
'email': email,
'role': role.value,
'is_admin': user.is_admin
}
}
def get_users(self) -> Dict[str, Any]:
"""
获取用户列表
Returns:
Dict[str, Any]: 用户列表
"""
users_info = []
for user_id, user in self.users.items():
users_info.append({
'id': user.id,
'login': user.login,
'name': user.name,
'email': user.email,
'role': user.role.value,
'is_admin': user.is_admin,
'is_disabled': user.is_disabled,
'last_seen_at': user.last_seen_at.isoformat() if user.last_seen_at else None,
'created_at': user.created_at.isoformat(),
'org_id': user.org_id,
'theme': user.theme,
'timezone': user.timezone
})
return {
'status': 'success',
'users': users_info,
'total': len(users_info)
}
def query_datasource(self, datasource_id: str, query: str,
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None) -> Dict[str, Any]:
"""
查询数据源
Args:
datasource_id: 数据源ID
query: 查询表达式
start_time: 开始时间
end_time: 结束时间
Returns:
Dict[str, Any]: 查询结果
"""
if datasource_id not in self.datasources:
return {'status': 'error', 'message': f'Datasource {datasource_id} not found'}
datasource = self.datasources[datasource_id]
if start_time is None:
start_time = datetime.now() - timedelta(hours=1)
if end_time is None:
end_time = datetime.now()
# 模拟查询结果
if datasource.type == DataSourceType.PROMETHEUS:
# 模拟Prometheus查询结果
result_data = self._simulate_prometheus_query(query, start_time, end_time)
else:
# 其他数据源的模拟结果
result_data = self._simulate_generic_query(query, start_time, end_time)
return {
'status': 'success',
'datasource': {
'id': datasource.id,
'name': datasource.name,
'type': datasource.type.value
},
'query': query,
'time_range': {
'from': start_time.isoformat(),
'to': end_time.isoformat()
},
'data': result_data
}
def _simulate_prometheus_query(self, query: str, start_time: datetime,
end_time: datetime) -> List[Dict[str, Any]]:
"""
模拟Prometheus查询结果
Args:
query: 查询表达式
start_time: 开始时间
end_time: 结束时间
Returns:
List[Dict[str, Any]]: 查询结果
"""
# 生成时间序列数据点
duration = end_time - start_time
points_count = min(100, max(10, int(duration.total_seconds() / 60))) # 每分钟一个点
time_points = []
for i in range(points_count):
timestamp = start_time + timedelta(seconds=i * duration.total_seconds() / points_count)
time_points.append(timestamp)
# 根据查询类型生成不同的数据
if 'cpu' in query.lower():
# CPU使用率数据
return [{
'metric': {'instance': 'localhost:9100', 'job': 'node-exporter'},
'values': [[tp.timestamp(), str(random.uniform(10, 90))] for tp in time_points]
}]
elif 'memory' in query.lower():
# 内存使用率数据
return [{
'metric': {'instance': 'localhost:9100', 'job': 'node-exporter'},
'values': [[tp.timestamp(), str(random.uniform(30, 80))] for tp in time_points]
}]
elif 'up' in query.lower():
# 服务状态数据
return [{
'metric': {'instance': 'localhost:9090', 'job': 'prometheus'},
'values': [[tp.timestamp(), '1'] for tp in time_points]
}]
else:
# 通用数据
return [{
'metric': {'__name__': 'generic_metric'},
'values': [[tp.timestamp(), str(random.uniform(0, 100))] for tp in time_points]
}]
def _simulate_generic_query(self, query: str, start_time: datetime,
end_time: datetime) -> List[Dict[str, Any]]:
"""
模拟通用查询结果
Args:
query: 查询表达式
start_time: 开始时间
end_time: 结束时间
Returns:
List[Dict[str, Any]]: 查询结果
"""
# 生成简单的表格数据
return [
{
'columns': [
{'text': 'Time', 'type': 'time'},
{'text': 'Value', 'type': 'number'}
],
'rows': [
[start_time.timestamp() * 1000, random.uniform(0, 100)],
[end_time.timestamp() * 1000, random.uniform(0, 100)]
],
'type': 'table'
}
]
def get_server_status(self) -> Dict[str, Any]:
"""
获取服务器状态
Returns:
Dict[str, Any]: 服务器状态
"""
return {
'server_id': self.server_id,
'status': 'running',
'version': self.version,
'build_info': self.build_info,
'database': {
'type': 'sqlite3',
'version': '3.39.0'
},
'stats': {
'total_dashboards': len(self.dashboards),
'total_datasources': len(self.datasources),
'total_users': len(self.users),
'total_organizations': len(self.organizations),
'total_folders': len(self.folders)
},
'settings': {
'default_theme': 'dark',
'default_timezone': 'browser',
'allow_sign_up': False,
'allow_org_create': False
},
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Grafana服务器
grafana = GrafanaServer("prod-grafana")
print("=== Grafana可视化服务器示例 ===")
# 添加数据源
print("\n=== 添加数据源 ===")
ds_result = grafana.add_datasource(
name="InfluxDB",
ds_type=DataSourceType.INFLUXDB,
url="http://localhost:8086",
database="telegraf",
user="admin",
password="admin123"
)
print(f"添加数据源结果: {ds_result}")
# 创建仪表板
print("\n=== 创建仪表板 ===")
dashboard_result = grafana.create_dashboard(
title="Application Monitoring",
description="Application performance monitoring dashboard",
tags=["application", "performance"]
)
print(f"创建仪表板结果: {dashboard_result}")
# 向仪表板添加面板
print("\n=== 添加面板 ===")
panel_result = grafana.add_panel_to_dashboard(
dashboard_uid=dashboard_result['dashboard']['uid'],
panel_title="Response Time",
panel_type=PanelType.GRAPH,
grid_pos={'h': 8, 'w': 12, 'x': 0, 'y': 0},
targets=[
{
'expr': 'http_request_duration_seconds',
'legend_format': '{{method}} {{status}}',
'ref_id': 'A'
}
],
datasource="prometheus-1"
)
print(f"添加面板结果: {panel_result}")
# 查询数据源
print("\n=== 查询数据源 ===")
query_result = grafana.query_datasource(
datasource_id="prometheus-1",
query="up",
start_time=datetime.now() - timedelta(hours=1),
end_time=datetime.now()
)
print(f"查询结果状态: {query_result['status']}")
print(f"数据点数量: {len(query_result['data'][0]['values']) if query_result['data'] else 0}")
# 创建用户
print("\n=== 创建用户 ===")
user_result = grafana.create_user(
login="developer",
name="Developer User",
email="dev@example.com",
role=UserRole.EDITOR
)
print(f"创建用户结果: {user_result}")
# 获取仪表板列表
print("\n=== 仪表板列表 ===")
dashboards_info = grafana.get_dashboards()
print(f"仪表板总数: {dashboards_info['total']}")
for dashboard in dashboards_info['dashboards']:
print(f" - {dashboard['title']} ({dashboard['uid']}): {len(dashboard['tags'])} tags")
# 获取数据源列表
print("\n=== 数据源列表 ===")
datasources_info = grafana.get_datasources()
print(f"数据源总数: {datasources_info['total']}")
for ds in datasources_info['datasources']:
print(f" - {ds['name']} ({ds['type']}): {ds['url']}")
# 获取用户列表
print("\n=== 用户列表 ===")
users_info = grafana.get_users()
print(f"用户总数: {users_info['total']}")
for user in users_info['users']:
print(f" - {user['name']} ({user['login']}): {user['role']}")
# 获取服务器状态
print("\n=== 服务器状态 ===")
server_status = grafana.get_server_status()
print(f"服务器ID: {server_status['server_id']}")
print(f"版本: {server_status['version']}")
print(f"状态: {server_status['status']}")
print(f"仪表板数: {server_status['stats']['total_dashboards']}")
print(f"数据源数: {server_status['stats']['total_datasources']}")
print(f"用户数: {server_status['stats']['total_users']}")
7. 工作流调度组件
7.1 Apache Airflow详解
from typing import Dict, List, Any, Optional, Tuple, Union, Callable
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import copy
class TaskState(Enum):
"""任务状态"""
NONE = "无状态"
SCHEDULED = "已调度"
QUEUED = "队列中"
RUNNING = "运行中"
SUCCESS = "成功"
FAILED = "失败"
UP_FOR_RETRY = "等待重试"
UP_FOR_RESCHEDULE = "等待重新调度"
UPSTREAM_FAILED = "上游失败"
SKIPPED = "跳过"
REMOVED = "已移除"
class DagState(Enum):
"""DAG状态"""
RUNNING = "运行中"
SUCCESS = "成功"
FAILED = "失败"
PAUSED = "暂停"
QUEUED = "队列中"
class TriggerRule(Enum):
"""触发规则"""
ALL_SUCCESS = "all_success" # 所有上游任务成功
ALL_FAILED = "all_failed" # 所有上游任务失败
ALL_DONE = "all_done" # 所有上游任务完成
ONE_SUCCESS = "one_success" # 至少一个上游任务成功
ONE_FAILED = "one_failed" # 至少一个上游任务失败
NONE_FAILED = "none_failed" # 没有上游任务失败
NONE_SKIPPED = "none_skipped" # 没有上游任务跳过
DUMMY = "dummy" # 总是触发
class ScheduleInterval(Enum):
"""调度间隔"""
ONCE = "@once" # 只执行一次
HOURLY = "@hourly" # 每小时
DAILY = "@daily" # 每天
WEEKLY = "@weekly" # 每周
MONTHLY = "@monthly" # 每月
YEARLY = "@yearly" # 每年
NONE = None # 不调度
class ExecutorType(Enum):
"""执行器类型"""
SEQUENTIAL = "SequentialExecutor"
LOCAL = "LocalExecutor"
CELERY = "CeleryExecutor"
KUBERNETES = "KubernetesExecutor"
DEBUG = "DebugExecutor"
@dataclass
class TaskInstance:
"""任务实例"""
task_id: str
dag_id: str
execution_date: datetime
state: TaskState = TaskState.NONE
start_date: Optional[datetime] = None
end_date: Optional[datetime] = None
duration: Optional[float] = None
try_number: int = 1
max_tries: int = 1
hostname: str = ""
unixname: str = ""
job_id: Optional[int] = None
pool: str = "default_pool"
queue: str = "default"
priority_weight: int = 1
operator: str = ""
queued_dttm: Optional[datetime] = None
pid: Optional[int] = None
executor_config: Dict[str, Any] = field(default_factory=dict)
log_url: str = ""
@dataclass
class Task:
"""任务定义"""
task_id: str
dag_id: str
operator_class: str
operator_kwargs: Dict[str, Any] = field(default_factory=dict)
upstream_task_ids: List[str] = field(default_factory=list)
downstream_task_ids: List[str] = field(default_factory=list)
trigger_rule: TriggerRule = TriggerRule.ALL_SUCCESS
depends_on_past: bool = False
wait_for_downstream: bool = False
retries: int = 0
retry_delay: timedelta = timedelta(minutes=5)
retry_exponential_backoff: bool = False
max_retry_delay: Optional[timedelta] = None
start_date: Optional[datetime] = None
end_date: Optional[datetime] = None
schedule_interval: Optional[str] = None
pool: str = "default_pool"
queue: str = "default"
priority_weight: int = 1
weight_rule: str = "downstream" # downstream, upstream, absolute
sla: Optional[timedelta] = None
execution_timeout: Optional[timedelta] = None
on_failure_callback: Optional[Callable] = None
on_success_callback: Optional[Callable] = None
on_retry_callback: Optional[Callable] = None
email_on_failure: bool = True
email_on_retry: bool = True
email: List[str] = field(default_factory=list)
@dataclass
class DagRun:
"""DAG运行实例"""
dag_id: str
run_id: str
execution_date: datetime
start_date: Optional[datetime] = None
end_date: Optional[datetime] = None
state: DagState = DagState.RUNNING
run_type: str = "scheduled" # scheduled, manual, backfill
external_trigger: bool = False
conf: Dict[str, Any] = field(default_factory=dict)
data_interval_start: Optional[datetime] = None
data_interval_end: Optional[datetime] = None
last_scheduling_decision: Optional[datetime] = None
dag_hash: Optional[str] = None
creating_job_id: Optional[int] = None
queued_at: Optional[datetime] = None
@dataclass
class Dag:
"""DAG定义"""
dag_id: str
description: str = ""
schedule_interval: Optional[str] = ScheduleInterval.DAILY.value
start_date: Optional[datetime] = None
end_date: Optional[datetime] = None
catchup: bool = True
max_active_runs: int = 16
max_active_tasks: int = 16
default_args: Dict[str, Any] = field(default_factory=dict)
params: Dict[str, Any] = field(default_factory=dict)
tags: List[str] = field(default_factory=list)
tasks: Dict[str, Task] = field(default_factory=dict)
is_paused: bool = False
is_subdag: bool = False
fileloc: str = ""
template_searchpath: Optional[List[str]] = None
template_undefined: str = "strict" # strict, jinja2.Undefined
user_defined_macros: Optional[Dict[str, Any]] = None
user_defined_filters: Optional[Dict[str, Any]] = None
default_view: str = "tree" # tree, graph, duration, gantt, landing_times
orientation: str = "LR" # LR, TB, RL, BT
concurrency: int = 16
max_active_runs_per_dag: int = 16
dagrun_timeout: Optional[timedelta] = None
sla_miss_callback: Optional[Callable] = None
default_view: str = "tree"
access_control: Dict[str, Any] = field(default_factory=dict)
doc_md: Optional[str] = None
@dataclass
class Connection:
"""连接配置"""
conn_id: str
conn_type: str
description: str = ""
host: Optional[str] = None
login: Optional[str] = None
password: Optional[str] = None
schema: Optional[str] = None
port: Optional[int] = None
extra: Dict[str, Any] = field(default_factory=dict)
uri: Optional[str] = None
is_encrypted: bool = False
is_extra_encrypted: bool = False
@dataclass
class Variable:
"""变量"""
key: str
val: str
description: str = ""
is_encrypted: bool = False
@dataclass
class Pool:
"""资源池"""
pool: str
slots: int
description: str = """
include_deferred: bool = True
class AirflowScheduler:
"""
Airflow调度器
"""
def __init__(self, scheduler_id: str = "airflow-scheduler-1"):
self.scheduler_id = scheduler_id
self.dags = {} # dag_id -> Dag
self.dag_runs = {} # run_id -> DagRun
self.task_instances = {} # (dag_id, task_id, execution_date) -> TaskInstance
self.connections = {} # conn_id -> Connection
self.variables = {} # key -> Variable
self.pools = {} # pool_name -> Pool
# 调度器状态
self.is_running = False
self.executor_type = ExecutorType.LOCAL
self.max_threads = 4
self.heartbeat_interval = 5 # 秒
self.dag_dir_list_interval = 300 # 秒
self.child_process_timeout = 60 # 秒
self.zombie_task_threshold = 300 # 秒
# 线程锁
self.dag_lock = threading.Lock()
self.task_lock = threading.Lock()
self.run_lock = threading.Lock()
# 调度器线程
self.scheduler_thread = None
self.heartbeat_thread = None
# 统计信息
self.stats = {
'total_dags': 0,
'active_dags': 0,
'paused_dags': 0,
'total_dag_runs': 0,
'running_dag_runs': 0,
'total_task_instances': 0,
'running_task_instances': 0,
'successful_task_instances': 0,
'failed_task_instances': 0
}
# 初始化默认数据
self._initialize_default_data()
def _initialize_default_data(self):
"""初始化默认数据"""
# 创建默认资源池
default_pool = Pool(
pool="default_pool",
slots=128,
description="Default pool"
)
self.pools["default_pool"] = default_pool
# 创建默认连接
local_mysql = Connection(
conn_id="mysql_default",
conn_type="mysql",
description="Default MySQL connection",
host="localhost",
login="airflow",
password="airflow",
schema="airflow",
port=3306
)
self.connections["mysql_default"] = local_mysql
# 创建示例DAG
self._create_example_dags()
def _create_example_dags(self):
"""创建示例DAG"""
# 数据处理DAG
data_processing_dag = Dag(
dag_id="data_processing_pipeline",
description="Daily data processing pipeline",
schedule_interval=ScheduleInterval.DAILY.value,
start_date=datetime(2024, 1, 1),
catchup=False,
max_active_runs=1,
tags=["data", "etl", "daily"]
)
# 添加任务
extract_task = Task(
task_id="extract_data",
dag_id="data_processing_pipeline",
operator_class="BashOperator",
operator_kwargs={
'bash_command': 'echo "Extracting data from source systems..."'
},
retries=2,
retry_delay=timedelta(minutes=5)
)
transform_task = Task(
task_id="transform_data",
dag_id="data_processing_pipeline",
operator_class="PythonOperator",
operator_kwargs={
'python_callable': 'transform_data_function'
},
upstream_task_ids=["extract_data"],
retries=1,
retry_delay=timedelta(minutes=3)
)
load_task = Task(
task_id="load_data",
dag_id="data_processing_pipeline",
operator_class="SqlOperator",
operator_kwargs={
'sql': 'INSERT INTO target_table SELECT * FROM staging_table',
'conn_id': 'mysql_default'
},
upstream_task_ids=["transform_data"],
retries=3,
retry_delay=timedelta(minutes=2)
)
validate_task = Task(
task_id="validate_data",
dag_id="data_processing_pipeline",
operator_class="PythonOperator",
operator_kwargs={
'python_callable': 'validate_data_quality'
},
upstream_task_ids=["load_data"]
)
# 设置任务依赖关系
transform_task.upstream_task_ids = ["extract_data"]
load_task.upstream_task_ids = ["transform_data"]
validate_task.upstream_task_ids = ["load_data"]
extract_task.downstream_task_ids = ["transform_data"]
transform_task.downstream_task_ids = ["load_data"]
load_task.downstream_task_ids = ["validate_data"]
data_processing_dag.tasks = {
"extract_data": extract_task,
"transform_data": transform_task,
"load_data": load_task,
"validate_data": validate_task
}
self.dags["data_processing_pipeline"] = data_processing_dag
# 机器学习训练DAG
ml_training_dag = Dag(
dag_id="ml_model_training",
description="Machine learning model training pipeline",
schedule_interval=ScheduleInterval.WEEKLY.value,
start_date=datetime(2024, 1, 1),
catchup=False,
max_active_runs=1,
tags=["ml", "training", "weekly"]
)
# 添加ML任务
prepare_data_task = Task(
task_id="prepare_training_data",
dag_id="ml_model_training",
operator_class="PythonOperator",
operator_kwargs={
'python_callable': 'prepare_training_data'
}
)
train_model_task = Task(
task_id="train_model",
dag_id="ml_model_training",
operator_class="PythonOperator",
operator_kwargs={
'python_callable': 'train_ml_model'
},
upstream_task_ids=["prepare_training_data"],
execution_timeout=timedelta(hours=2)
)
evaluate_model_task = Task(
task_id="evaluate_model",
dag_id="ml_model_training",
operator_class="PythonOperator",
operator_kwargs={
'python_callable': 'evaluate_model_performance'
},
upstream_task_ids=["train_model"]
)
deploy_model_task = Task(
task_id="deploy_model",
dag_id="ml_model_training",
operator_class="BashOperator",
operator_kwargs={
'bash_command': 'echo "Deploying model to production..."'
},
upstream_task_ids=["evaluate_model"],
trigger_rule=TriggerRule.ALL_SUCCESS
)
# 设置任务依赖关系
train_model_task.upstream_task_ids = ["prepare_training_data"]
evaluate_model_task.upstream_task_ids = ["train_model"]
deploy_model_task.upstream_task_ids = ["evaluate_model"]
prepare_data_task.downstream_task_ids = ["train_model"]
train_model_task.downstream_task_ids = ["evaluate_model"]
evaluate_model_task.downstream_task_ids = ["deploy_model"]
ml_training_dag.tasks = {
"prepare_training_data": prepare_data_task,
"train_model": train_model_task,
"evaluate_model": evaluate_model_task,
"deploy_model": deploy_model_task
}
self.dags["ml_model_training"] = ml_training_dag
def add_dag(self, dag: Dag) -> Dict[str, Any]:
"""
添加DAG
Args:
dag: DAG对象
Returns:
Dict[str, Any]: 添加结果
"""
if dag.dag_id in self.dags:
return {'status': 'error', 'message': f'DAG {dag.dag_id} already exists'}
with self.dag_lock:
self.dags[dag.dag_id] = dag
self.stats['total_dags'] += 1
if not dag.is_paused:
self.stats['active_dags'] += 1
else:
self.stats['paused_dags'] += 1
return {
'status': 'success',
'dag': {
'dag_id': dag.dag_id,
'description': dag.description,
'schedule_interval': dag.schedule_interval,
'is_paused': dag.is_paused,
'task_count': len(dag.tasks)
}
}
def get_dag(self, dag_id: str) -> Dict[str, Any]:
"""
获取DAG信息
Args:
dag_id: DAG ID
Returns:
Dict[str, Any]: DAG信息
"""
if dag_id not in self.dags:
return {'status': 'error', 'message': f'DAG {dag_id} not found'}
dag = self.dags[dag_id]
# 获取任务信息
tasks_info = []
for task_id, task in dag.tasks.items():
tasks_info.append({
'task_id': task.task_id,
'operator_class': task.operator_class,
'upstream_task_ids': task.upstream_task_ids,
'downstream_task_ids': task.downstream_task_ids,
'trigger_rule': task.trigger_rule.value,
'retries': task.retries,
'pool': task.pool,
'queue': task.queue
})
return {
'status': 'success',
'dag': {
'dag_id': dag.dag_id,
'description': dag.description,
'schedule_interval': dag.schedule_interval,
'start_date': dag.start_date.isoformat() if dag.start_date else None,
'end_date': dag.end_date.isoformat() if dag.end_date else None,
'catchup': dag.catchup,
'max_active_runs': dag.max_active_runs,
'max_active_tasks': dag.max_active_tasks,
'is_paused': dag.is_paused,
'tags': dag.tags,
'tasks': tasks_info,
'task_count': len(dag.tasks)
}
}
def trigger_dag(self, dag_id: str, run_id: Optional[str] = None,
conf: Optional[Dict[str, Any]] = None,
execution_date: Optional[datetime] = None) -> Dict[str, Any]:
"""
触发DAG运行
Args:
dag_id: DAG ID
run_id: 运行ID
conf: 配置参数
execution_date: 执行日期
Returns:
Dict[str, Any]: 触发结果
"""
if dag_id not in self.dags:
return {'status': 'error', 'message': f'DAG {dag_id} not found'}
dag = self.dags[dag_id]
if dag.is_paused:
return {'status': 'error', 'message': f'DAG {dag_id} is paused'}
if execution_date is None:
execution_date = datetime.now()
if run_id is None:
run_id = f"manual__{execution_date.strftime('%Y-%m-%dT%H:%M:%S')}"
# 检查是否已存在相同的运行
if run_id in self.dag_runs:
return {'status': 'error', 'message': f'DagRun {run_id} already exists'}
# 创建DAG运行实例
dag_run = DagRun(
dag_id=dag_id,
run_id=run_id,
execution_date=execution_date,
start_date=datetime.now(),
state=DagState.RUNNING,
run_type="manual",
external_trigger=True,
conf=conf or {},
data_interval_start=execution_date,
data_interval_end=execution_date + timedelta(days=1)
)
with self.run_lock:
self.dag_runs[run_id] = dag_run
self.stats['total_dag_runs'] += 1
self.stats['running_dag_runs'] += 1
# 创建任务实例
self._create_task_instances(dag, dag_run)
return {
'status': 'success',
'dag_run': {
'dag_id': dag_id,
'run_id': run_id,
'execution_date': execution_date.isoformat(),
'state': dag_run.state.value,
'run_type': dag_run.run_type
}
}
def _create_task_instances(self, dag: Dag, dag_run: DagRun):
"""
为DAG运行创建任务实例
Args:
dag: DAG对象
dag_run: DAG运行实例
"""
with self.task_lock:
for task_id, task in dag.tasks.items():
task_instance = TaskInstance(
task_id=task_id,
dag_id=dag.dag_id,
execution_date=dag_run.execution_date,
state=TaskState.SCHEDULED,
max_tries=task.retries + 1,
pool=task.pool,
queue=task.queue,
priority_weight=task.priority_weight,
operator=task.operator_class
)
key = (dag.dag_id, task_id, dag_run.execution_date)
self.task_instances[key] = task_instance
self.stats['total_task_instances'] += 1
def get_dag_runs(self, dag_id: Optional[str] = None,
state: Optional[DagState] = None,
limit: int = 100) -> Dict[str, Any]:
"""
获取DAG运行列表
Args:
dag_id: DAG ID过滤
state: 状态过滤
limit: 限制数量
Returns:
Dict[str, Any]: DAG运行列表
"""
dag_runs_info = []
count = 0
for run_id, dag_run in self.dag_runs.items():
if count >= limit:
break
# 应用过滤条件
if dag_id is not None and dag_run.dag_id != dag_id:
continue
if state is not None and dag_run.state != state:
continue
dag_runs_info.append({
'dag_id': dag_run.dag_id,
'run_id': dag_run.run_id,
'execution_date': dag_run.execution_date.isoformat(),
'start_date': dag_run.start_date.isoformat() if dag_run.start_date else None,
'end_date': dag_run.end_date.isoformat() if dag_run.end_date else None,
'state': dag_run.state.value,
'run_type': dag_run.run_type,
'external_trigger': dag_run.external_trigger,
'conf': dag_run.conf
})
count += 1
return {
'status': 'success',
'dag_runs': dag_runs_info,
'total': len(dag_runs_info)
}
def get_task_instances(self, dag_id: str, run_id: str) -> Dict[str, Any]:
"""
获取任务实例列表
Args:
dag_id: DAG ID
run_id: 运行ID
Returns:
Dict[str, Any]: 任务实例列表
"""
if run_id not in self.dag_runs:
return {'status': 'error', 'message': f'DagRun {run_id} not found'}
dag_run = self.dag_runs[run_id]
task_instances_info = []
for key, task_instance in self.task_instances.items():
ti_dag_id, ti_task_id, ti_execution_date = key
if (ti_dag_id == dag_id and
ti_execution_date == dag_run.execution_date):
task_instances_info.append({
'task_id': task_instance.task_id,
'dag_id': task_instance.dag_id,
'execution_date': task_instance.execution_date.isoformat(),
'state': task_instance.state.value,
'start_date': task_instance.start_date.isoformat() if task_instance.start_date else None,
'end_date': task_instance.end_date.isoformat() if task_instance.end_date else None,
'duration': task_instance.duration,
'try_number': task_instance.try_number,
'max_tries': task_instance.max_tries,
'operator': task_instance.operator,
'pool': task_instance.pool,
'queue': task_instance.queue,
'priority_weight': task_instance.priority_weight
})
return {
'status': 'success',
'task_instances': task_instances_info,
'total': len(task_instances_info)
}
def run_task_instance(self, dag_id: str, task_id: str,
execution_date: datetime) -> Dict[str, Any]:
"""
运行任务实例
Args:
dag_id: DAG ID
task_id: 任务ID
execution_date: 执行日期
Returns:
Dict[str, Any]: 运行结果
"""
key = (dag_id, task_id, execution_date)
if key not in self.task_instances:
return {'status': 'error', 'message': f'TaskInstance {key} not found'}
task_instance = self.task_instances[key]
if task_instance.state not in [TaskState.SCHEDULED, TaskState.QUEUED, TaskState.UP_FOR_RETRY]:
return {'status': 'error', 'message': f'TaskInstance {key} is not in a runnable state'}
# 模拟任务执行
with self.task_lock:
task_instance.state = TaskState.RUNNING
task_instance.start_date = datetime.now()
task_instance.hostname = "airflow-worker-1"
task_instance.pid = random.randint(1000, 9999)
self.stats['running_task_instances'] += 1
# 模拟任务执行时间
execution_time = random.uniform(1, 10) # 1-10秒
time.sleep(execution_time)
# 模拟任务结果（90%成功率）
success = random.random() > 0.1
with self.task_lock:
task_instance.end_date = datetime.now()
task_instance.duration = execution_time
if success:
task_instance.state = TaskState.SUCCESS
self.stats['successful_task_instances'] += 1
else:
if task_instance.try_number < task_instance.max_tries:
task_instance.state = TaskState.UP_FOR_RETRY
task_instance.try_number += 1
else:
task_instance.state = TaskState.FAILED
self.stats['failed_task_instances'] += 1
self.stats['running_task_instances'] -= 1
return {
'status': 'success',
'task_instance': {
'task_id': task_instance.task_id,
'dag_id': task_instance.dag_id,
'execution_date': task_instance.execution_date.isoformat(),
'state': task_instance.state.value,
'duration': task_instance.duration,
'try_number': task_instance.try_number
}
}
def pause_dag(self, dag_id: str) -> Dict[str, Any]:
"""
暂停DAG
Args:
dag_id: DAG ID
Returns:
Dict[str, Any]: 暂停结果
"""
if dag_id not in self.dags:
return {'status': 'error', 'message': f'DAG {dag_id} not found'}
dag = self.dags[dag_id]
if dag.is_paused:
return {'status': 'error', 'message': f'DAG {dag_id} is already paused'}
with self.dag_lock:
dag.is_paused = True
self.stats['active_dags'] -= 1
self.stats['paused_dags'] += 1
return {
'status': 'success',
'message': f'DAG {dag_id} paused successfully',
'dag_id': dag_id,
'is_paused': True
}
def unpause_dag(self, dag_id: str) -> Dict[str, Any]:
"""
取消暂停DAG
Args:
dag_id: DAG ID
Returns:
Dict[str, Any]: 取消暂停结果
"""
if dag_id not in self.dags:
return {'status': 'error', 'message': f'DAG {dag_id} not found'}
dag = self.dags[dag_id]
if not dag.is_paused:
return {'status': 'error', 'message': f'DAG {dag_id} is not paused'}
with self.dag_lock:
dag.is_paused = False
self.stats['paused_dags'] -= 1
self.stats['active_dags'] += 1
return {
'status': 'success',
'message': f'DAG {dag_id} unpaused successfully',
'dag_id': dag_id,
'is_paused': False
}
def add_connection(self, connection: Connection) -> Dict[str, Any]:
"""
添加连接
Args:
connection: 连接对象
Returns:
Dict[str, Any]: 添加结果
"""
if connection.conn_id in self.connections:
return {'status': 'error', 'message': f'Connection {connection.conn_id} already exists'}
self.connections[connection.conn_id] = connection
return {
'status': 'success',
'connection': {
'conn_id': connection.conn_id,
'conn_type': connection.conn_type,
'host': connection.host,
'port': connection.port,
'schema': connection.schema
}
}
def get_connections(self) -> Dict[str, Any]:
"""
获取连接列表
Returns:
Dict[str, Any]: 连接列表
"""
connections_info = []
for conn_id, connection in self.connections.items():
connections_info.append({
'conn_id': connection.conn_id,
'conn_type': connection.conn_type,
'description': connection.description,
'host': connection.host,
'login': connection.login,
'schema': connection.schema,
'port': connection.port,
'is_encrypted': connection.is_encrypted
})
return {
'status': 'success',
'connections': connections_info,
'total': len(connections_info)
}
def set_variable(self, key: str, value: str, description: str = "") -> Dict[str, Any]:
"""
设置变量
Args:
key: 变量键
value: 变量值
description: 描述
Returns:
Dict[str, Any]: 设置结果
"""
variable = Variable(
key=key,
val=value,
description=description
)
self.variables[key] = variable
return {
'status': 'success',
'variable': {
'key': key,
'value': value,
'description': description
}
}
def get_variable(self, key: str) -> Dict[str, Any]:
"""
获取变量
Args:
key: 变量键
Returns:
Dict[str, Any]: 变量信息
"""
if key not in self.variables:
return {'status': 'error', 'message': f'Variable {key} not found'}
variable = self.variables[key]
return {
'status': 'success',
'variable': {
'key': variable.key,
'value': variable.val,
'description': variable.description,
'is_encrypted': variable.is_encrypted
}
}
def get_scheduler_status(self) -> Dict[str, Any]:
"""
获取调度器状态
Returns:
Dict[str, Any]: 调度器状态
"""
return {
'scheduler_id': self.scheduler_id,
'is_running': self.is_running,
'executor_type': self.executor_type.value,
'max_threads': self.max_threads,
'heartbeat_interval': self.heartbeat_interval,
'stats': self.stats,
'config': {
'dag_dir_list_interval': self.dag_dir_list_interval,
'child_process_timeout': self.child_process_timeout,
'zombie_task_threshold': self.zombie_task_threshold
},
'pools': {
pool_name: {
'slots': pool.slots,
'description': pool.description
} for pool_name, pool in self.pools.items()
},
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Airflow调度器
scheduler = AirflowScheduler("prod-scheduler")
print("=== Apache Airflow调度器示例 ===")
# 获取DAG信息
print("\n=== DAG信息 ===")
dag_info = scheduler.get_dag("data_processing_pipeline")
if dag_info['status'] == 'success':
dag = dag_info['dag']
print(f"DAG ID: {dag['dag_id']}")
print(f"描述: {dag['description']}")
print(f"调度间隔: {dag['schedule_interval']}")
print(f"任务数量: {dag['task_count']}")
print(f"标签: {', '.join(dag['tags'])}")
print("任务列表:")
for task in dag['tasks']:
print(f" - {task['task_id']} ({task['operator_class']})")
if task['upstream_task_ids']:
print(f" 上游任务: {', '.join(task['upstream_task_ids'])}")
# 触发DAG运行
print("\n=== 触发DAG运行 ===")
trigger_result = scheduler.trigger_dag(
dag_id="data_processing_pipeline",
conf={'env': 'production', 'batch_size': 1000}
)
print(f"触发结果: {trigger_result['status']}")
if trigger_result['status'] == 'success':
dag_run = trigger_result['dag_run']
print(f"运行ID: {dag_run['run_id']}")
print(f"执行日期: {dag_run['execution_date']}")
print(f"状态: {dag_run['state']}")
# 获取任务实例
print("\n=== 任务实例 ===")
if trigger_result['status'] == 'success':
run_id = trigger_result['dag_run']['run_id']
task_instances = scheduler.get_task_instances("data_processing_pipeline", run_id)
print(f"任务实例总数: {task_instances['total']}")
for ti in task_instances['task_instances']:
print(f" - {ti['task_id']}: {ti['state']} (尝试次数: {ti['try_number']}/{ti['max_tries']})")
# 运行任务实例
print("\n=== 运行任务实例 ===")
if trigger_result['status'] == 'success':
execution_date = datetime.fromisoformat(trigger_result['dag_run']['execution_date'])
# 运行extract_data任务
run_result = scheduler.run_task_instance(
dag_id="data_processing_pipeline",
task_id="extract_data",
execution_date=execution_date
)
print(f"运行extract_data任务: {run_result['status']}")
if run_result['status'] == 'success':
ti = run_result['task_instance']
print(f" 状态: {ti['state']}")
print(f" 执行时间: {ti['duration']:.2f}秒")
# 添加连接
print("\n=== 添加连接 ===")
postgres_conn = Connection(
conn_id="postgres_default",
conn_type="postgres",
description="Default PostgreSQL connection",
host="localhost",
login="postgres",
password="postgres",
schema="airflow",
port=5432
)
conn_result = scheduler.add_connection(postgres_conn)
print(f"添加连接结果: {conn_result}")
# 设置变量
print("\n=== 设置变量 ===")
var_result = scheduler.set_variable(
key="data_source_url",
value="https://api.example.com/data",
description="Data source API URL"
)
print(f"设置变量结果: {var_result}")
# 暂停DAG
print("\n=== 暂停DAG ===")
pause_result = scheduler.pause_dag("ml_model_training")
print(f"暂停DAG结果: {pause_result}")
# 获取DAG运行列表
print("\n=== DAG运行列表 ===")
dag_runs = scheduler.get_dag_runs(dag_id="data_processing_pipeline")
print(f"DAG运行总数: {dag_runs['total']}")
for run in dag_runs['dag_runs']:
print(f" - {run['run_id']}: {run['state']} ({run['run_type']})")
# 获取调度器状态
print("\n=== 调度器状态 ===")
status = scheduler.get_scheduler_status()
print(f"调度器ID: {status['scheduler_id']}")
print(f"运行状态: {status['is_running']}")
print(f"执行器类型: {status['executor_type']}")
print(f"最大线程数: {status['max_threads']}")
print("统计信息:")
for key, value in status['stats'].items():
print(f" {key}: {value}")
## 10. 生态系统集成与最佳实践
### 10.1 组件集成架构
```python
from enum import Enum
from dataclasses import dataclass
from typing import Dict, List, Optional, Any
from datetime import datetime
import json
class IntegrationType(Enum):
"""集成类型"""
DATA_PIPELINE = "data_pipeline"
REAL_TIME_ANALYTICS = "real_time_analytics"
BATCH_PROCESSING = "batch_processing"
MACHINE_LEARNING = "machine_learning"
DATA_WAREHOUSE = "data_warehouse"
class ComponentRole(Enum):
"""组件角色"""
DATA_SOURCE = "data_source"
DATA_INGESTION = "data_ingestion"
DATA_PROCESSING = "data_processing"
DATA_STORAGE = "data_storage"
DATA_QUERY = "data_query"
DATA_VISUALIZATION = "data_visualization"
WORKFLOW_ORCHESTRATION = "workflow_orchestration"
MONITORING = "monitoring"
@dataclass
class ComponentIntegration:
"""组件集成配置"""
component_name: str
role: ComponentRole
dependencies: List[str]
configuration: Dict[str, Any]
health_check_url: Optional[str] = None
metrics_endpoint: Optional[str] = None
@dataclass
class DataPipeline:
"""数据管道"""
pipeline_id: str
name: str
integration_type: IntegrationType
components: List[ComponentIntegration]
data_flow: List[Dict[str, str]]
schedule: Optional[str] = None
sla_minutes: Optional[int] = None
created_at: datetime = None
def __post_init__(self):
if self.created_at is None:
self.created_at = datetime.now()
class HadoopEcosystemIntegrator:
"""Hadoop生态系统集成器"""
def __init__(self):
self.pipelines: Dict[str, DataPipeline] = {}
self.component_registry: Dict[str, ComponentIntegration] = {}
self.integration_templates = self._initialize_templates()
def _initialize_templates(self) -> Dict[str, Dict[str, Any]]:
"""初始化集成模板"""
return {
"real_time_analytics": {
"components": [
"kafka", "storm", "hbase", "elasticsearch", "grafana"
],
"data_flow": [
{"from": "kafka", "to": "storm", "type": "stream"},
{"from": "storm", "to": "hbase", "type": "write"},
{"from": "hbase", "to": "elasticsearch", "type": "index"},
{"from": "elasticsearch", "to": "grafana", "type": "query"}
]
},
"batch_processing": {
"components": [
"hdfs", "yarn", "mapreduce", "hive", "oozie"
],
"data_flow": [
{"from": "hdfs", "to": "mapreduce", "type": "read"},
{"from": "mapreduce", "to": "hive", "type": "transform"},
{"from": "hive", "to": "hdfs", "type": "write"}
]
},
"data_warehouse": {
"components": [
"sqoop", "hdfs", "hive", "impala", "tableau"
],
"data_flow": [
{"from": "sqoop", "to": "hdfs", "type": "import"},
{"from": "hdfs", "to": "hive", "type": "load"},
{"from": "hive", "to": "impala", "type": "query"},
{"from": "impala", "to": "tableau", "type": "visualize"}
]
}
}
def register_component(self, component: ComponentIntegration) -> Dict[str, Any]:
"""注册组件"""
try:
# 验证依赖关系
for dep in component.dependencies:
if dep not in self.component_registry:
return {
'status': 'error',
'message': f'Dependency {dep} not found'
}
self.component_registry[component.component_name] = component
return {
'status': 'success',
'component_name': component.component_name,
'role': component.role.value,
'dependencies': component.dependencies
}
except Exception as e:
return {
'status': 'error',
'message': f'Failed to register component: {str(e)}'
}
def create_pipeline(self, pipeline_config: Dict[str, Any]) -> Dict[str, Any]:
"""创建数据管道"""
try:
pipeline_id = pipeline_config['pipeline_id']
integration_type = IntegrationType(pipeline_config['integration_type'])
# 获取模板
template = self.integration_templates.get(integration_type.value)
if not template:
return {
'status': 'error',
'message': f'No template found for {integration_type.value}'
}
# 构建组件列表
components = []
for comp_name in template['components']:
if comp_name in self.component_registry:
components.append(self.component_registry[comp_name])
else:
# 创建默认组件配置
default_component = ComponentIntegration(
component_name=comp_name,
role=self._get_default_role(comp_name),
dependencies=[],
configuration={}
)
components.append(default_component)
# 创建管道
pipeline = DataPipeline(
pipeline_id=pipeline_id,
name=pipeline_config.get('name', f'Pipeline-{pipeline_id}'),
integration_type=integration_type,
components=components,
data_flow=template['data_flow'],
schedule=pipeline_config.get('schedule'),
sla_minutes=pipeline_config.get('sla_minutes')
)
self.pipelines[pipeline_id] = pipeline
return {
'status': 'success',
'pipeline_id': pipeline_id,
'integration_type': integration_type.value,
'components_count': len(components),
'data_flow_steps': len(template['data_flow'])
}
except Exception as e:
return {
'status': 'error',
'message': f'Failed to create pipeline: {str(e)}'
}
def _get_default_role(self, component_name: str) -> ComponentRole:
"""获取组件默认角色"""
role_mapping = {
'kafka': ComponentRole.DATA_INGESTION,
'storm': ComponentRole.DATA_PROCESSING,
'spark': ComponentRole.DATA_PROCESSING,
'hdfs': ComponentRole.DATA_STORAGE,
'hbase': ComponentRole.DATA_STORAGE,
'hive': ComponentRole.DATA_QUERY,
'impala': ComponentRole.DATA_QUERY,
'elasticsearch': ComponentRole.DATA_STORAGE,
'grafana': ComponentRole.DATA_VISUALIZATION,
'sqoop': ComponentRole.DATA_INGESTION,
'flume': ComponentRole.DATA_INGESTION,
'oozie': ComponentRole.WORKFLOW_ORCHESTRATION,
'airflow': ComponentRole.WORKFLOW_ORCHESTRATION,
'ranger': ComponentRole.MONITORING,
'knox': ComponentRole.MONITORING
}
return role_mapping.get(component_name, ComponentRole.DATA_PROCESSING)
def validate_pipeline(self, pipeline_id: str) -> Dict[str, Any]:
"""验证管道配置"""
if pipeline_id not in self.pipelines:
return {
'status': 'error',
'message': f'Pipeline {pipeline_id} not found'
}
pipeline = self.pipelines[pipeline_id]
validation_results = []
# 验证组件依赖
for component in pipeline.components:
for dep in component.dependencies:
dep_exists = any(c.component_name == dep for c in pipeline.components)
if not dep_exists:
validation_results.append({
'type': 'dependency_missing',
'component': component.component_name,
'missing_dependency': dep
})
# 验证数据流
component_names = {c.component_name for c in pipeline.components}
for flow in pipeline.data_flow:
if flow['from'] not in component_names:
validation_results.append({
'type': 'flow_source_missing',
'flow': flow,
'missing_component': flow['from']
})
if flow['to'] not in component_names:
validation_results.append({
'type': 'flow_target_missing',
'flow': flow,
'missing_component': flow['to']
})
return {
'status': 'success' if not validation_results else 'warning',
'pipeline_id': pipeline_id,
'validation_results': validation_results,
'is_valid': len(validation_results) == 0
}
def get_pipeline_status(self, pipeline_id: str) -> Dict[str, Any]:
"""获取管道状态"""
if pipeline_id not in self.pipelines:
return {
'status': 'error',
'message': f'Pipeline {pipeline_id} not found'
}
pipeline = self.pipelines[pipeline_id]
# 模拟组件健康检查
component_health = []
for component in pipeline.components:
health_status = 'healthy' if component.component_name in ['hdfs', 'yarn', 'hive'] else 'unknown'
component_health.append({
'component': component.component_name,
'role': component.role.value,
'status': health_status,
'last_check': datetime.now().isoformat()
})
return {
'status': 'success',
'pipeline_id': pipeline_id,
'name': pipeline.name,
'integration_type': pipeline.integration_type.value,
'created_at': pipeline.created_at.isoformat(),
'components_count': len(pipeline.components),
'component_health': component_health,
'data_flow_steps': len(pipeline.data_flow)
}
def list_pipelines(self) -> Dict[str, Any]:
"""列出所有管道"""
pipelines_info = []
for pipeline_id, pipeline in self.pipelines.items():
pipelines_info.append({
'pipeline_id': pipeline_id,
'name': pipeline.name,
'integration_type': pipeline.integration_type.value,
'components_count': len(pipeline.components),
'created_at': pipeline.created_at.isoformat()
})
return {
'status': 'success',
'total_pipelines': len(pipelines_info),
'pipelines': pipelines_info
}
def get_integration_recommendations(self, requirements: Dict[str, Any]) -> Dict[str, Any]:
"""获取集成建议"""
data_volume = requirements.get('data_volume', 'medium')
latency_requirement = requirements.get('latency', 'batch')
data_types = requirements.get('data_types', ['structured'])
recommendations = []
# 基于需求推荐架构
if latency_requirement == 'real_time':
if 'streaming' in data_types:
recommendations.append({
'architecture': 'Lambda Architecture',
'components': ['kafka', 'storm', 'hbase', 'hdfs', 'spark'],
'description': '实时流处理架构，支持低延迟数据处理'
})
recommendations.append({
'architecture': 'Kappa Architecture',
'components': ['kafka', 'spark_streaming', 'elasticsearch'],
'description': '简化的流处理架构，统一批处理和流处理'
})
elif latency_requirement == 'batch':
recommendations.append({
'architecture': 'Traditional Batch Processing',
'components': ['hdfs', 'yarn', 'mapreduce', 'hive', 'oozie'],
'description': '传统批处理架构，适合大数据量离线分析'
})
# 基于数据量推荐
if data_volume == 'large':
recommendations.append({
'architecture': 'Distributed Data Warehouse',
'components': ['hdfs', 'hive', 'impala', 'kudu', 'spark'],
'description': '分布式数据仓库，支持大规模数据存储和查询'
})
return {
'status': 'success',
'requirements': requirements,
'recommendations': recommendations,
'total_recommendations': len(recommendations)
}
### 10.2 性能优化最佳实践
```python
class PerformanceOptimizer:
"""性能优化器"""
def __init__(self):
self.optimization_rules = self._initialize_rules()
def _initialize_rules(self) -> Dict[str, Dict[str, Any]]:
"""初始化优化规则"""
return {
'hdfs': {
'block_size': {
'small_files': '64MB',
'large_files': '256MB',
'very_large_files': '512MB'
},
'replication_factor': {
'development': 1,
'testing': 2,
'production': 3
},
'compression': {
'text_files': 'gzip',
'sequence_files': 'snappy',
'parquet_files': 'snappy'
}
},
'yarn': {
'memory_allocation': {
'container_min_mb': 1024,
'container_max_mb': 8192,
'am_memory_mb': 1024
},
'cpu_allocation': {
'container_min_vcores': 1,
'container_max_vcores': 4
}
},
'mapreduce': {
'map_tasks': {
'memory_mb': 2048,
'java_opts': '-Xmx1638m',
'io_sort_mb': 512
},
'reduce_tasks': {
'memory_mb': 4096,
'java_opts': '-Xmx3276m',
'parallel_copies': 10
}
},
'hive': {
'execution_engine': 'tez',
'vectorization': True,
'cost_based_optimizer': True,
'compression': {
'intermediate': 'snappy',
'final': 'gzip'
}
},
'spark': {
'executor_memory': '4g',
'executor_cores': 2,
'driver_memory': '2g',
'serializer': 'org.apache.spark.serializer.KryoSerializer',
'sql_adaptive_enabled': True
}
}
def analyze_performance(self, component: str, metrics: Dict[str, Any]) -> Dict[str, Any]:
"""分析性能指标"""
if component not in self.optimization_rules:
return {
'status': 'error',
'message': f'No optimization rules for component: {component}'
}
recommendations = []
# 基于组件类型分析
if component == 'hdfs':
recommendations.extend(self._analyze_hdfs_performance(metrics))
elif component == 'yarn':
recommendations.extend(self._analyze_yarn_performance(metrics))
elif component == 'mapreduce':
recommendations.extend(self._analyze_mapreduce_performance(metrics))
elif component == 'hive':
recommendations.extend(self._analyze_hive_performance(metrics))
elif component == 'spark':
recommendations.extend(self._analyze_spark_performance(metrics))
return {
'status': 'success',
'component': component,
'recommendations': recommendations,
'total_recommendations': len(recommendations)
}
def _analyze_hdfs_performance(self, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
"""分析HDFS性能"""
recommendations = []
# 检查小文件问题
if metrics.get('avg_file_size_mb', 0) < 64:
recommendations.append({
'type': 'small_files',
'priority': 'high',
'description': '存在大量小文件，建议合并文件或增加块大小',
'action': 'Merge small files or increase block size to 128MB'
})
# 检查磁盘使用率
if metrics.get('disk_usage_percent', 0) > 80:
recommendations.append({
'type': 'disk_usage',
'priority': 'high',
'description': '磁盘使用率过高，建议清理或扩容',
'action': 'Clean up old data or add more storage capacity'
})
# 检查网络带宽
if metrics.get('network_utilization_percent', 0) > 70:
recommendations.append({
'type': 'network',
'priority': 'medium',
'description': '网络带宽使用率高，建议优化数据本地性',
'action': 'Improve data locality or upgrade network infrastructure'
})
return recommendations
def _analyze_yarn_performance(self, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
"""分析YARN性能"""
recommendations = []
# 检查资源利用率
memory_utilization = metrics.get('memory_utilization_percent', 0)
if memory_utilization > 90:
recommendations.append({
'type': 'memory_pressure',
'priority': 'high',
'description': '内存使用率过高，建议调整容器大小或增加节点',
'action': 'Adjust container sizes or add more NodeManagers'
})
elif memory_utilization < 30:
recommendations.append({
'type': 'memory_underutilization',
'priority': 'medium',
'description': '内存利用率低，建议增加并发任务或调整资源配置',
'action': 'Increase parallelism or adjust resource allocation'
})
# 检查队列配置
if metrics.get('pending_applications', 0) > 10:
recommendations.append({
'type': 'queue_congestion',
'priority': 'medium',
'description': '队列拥堵，建议调整队列配置或增加资源',
'action': 'Adjust queue configuration or add more resources'
})
return recommendations
def _analyze_mapreduce_performance(self, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
"""分析MapReduce性能"""
recommendations = []
# 检查任务执行时间
avg_map_time = metrics.get('avg_map_time_seconds', 0)
if avg_map_time > 300: # 5分钟
recommendations.append({
'type': 'slow_map_tasks',
'priority': 'high',
'description': 'Map任务执行时间过长，建议优化输入分片或增加内存',
'action': 'Optimize input splits or increase map memory'
})
# 检查数据倾斜
task_time_variance = metrics.get('task_time_variance', 0)
if task_time_variance > 0.5:
recommendations.append({
'type': 'data_skew',
'priority': 'high',
'description': '存在数据倾斜，建议重新分区或使用自定义分区器',
'action': 'Repartition data or use custom partitioner'
})
return recommendations
def _analyze_hive_performance(self, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
"""分析Hive性能"""
recommendations = []
# 检查查询执行时间
avg_query_time = metrics.get('avg_query_time_seconds', 0)
if avg_query_time > 600: # 10分钟
recommendations.append({
'type': 'slow_queries',
'priority': 'high',
'description': '查询执行时间过长，建议优化SQL或创建索引',
'action': 'Optimize SQL queries or create appropriate indexes'
})
# 检查表格式
if metrics.get('orc_tables_percent', 0) < 50:
recommendations.append({
'type': 'table_format',
'priority': 'medium',
'description': '建议使用ORC格式提高查询性能',
'action': 'Convert tables to ORC format for better performance'
})
return recommendations
def _analyze_spark_performance(self, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
"""分析Spark性能"""
recommendations = []
# 检查GC时间
gc_time_percent = metrics.get('gc_time_percent', 0)
if gc_time_percent > 10:
recommendations.append({
'type': 'gc_pressure',
'priority': 'high',
'description': 'GC时间过长，建议调整内存配置或使用G1GC',
'action': 'Adjust memory settings or use G1 garbage collector'
})
# 检查数据序列化
if not metrics.get('kryo_serializer_enabled', False):
recommendations.append({
'type': 'serialization',
'priority': 'medium',
'description': '建议启用Kryo序列化器提高性能',
'action': 'Enable Kryo serializer for better performance'
})
return recommendations
### 10.3 监控和运维
```python
class EcosystemMonitor:
"""生态系统监控器"""
def __init__(self):
self.alerts = []
self.metrics_history = []
self.thresholds = self._initialize_thresholds()
def _initialize_thresholds(self) -> Dict[str, Dict[str, float]]:
"""初始化告警阈值"""
return {
'hdfs': {
'disk_usage_percent': 85.0,
'namenode_heap_usage_percent': 80.0,
'datanode_failed_volumes': 1.0
},
'yarn': {
'memory_utilization_percent': 90.0,
'nodemanager_unhealthy_percent': 10.0,
'application_failed_percent': 5.0
},
'hive': {
'metastore_connection_failures': 5.0,
'query_failure_rate_percent': 10.0,
'avg_query_time_seconds': 300.0
}
}
def collect_metrics(self, component: str) -> Dict[str, Any]:
"""收集组件指标"""
# 模拟指标收集
import random
if component == 'hdfs':
metrics = {
'disk_usage_percent': random.uniform(60, 95),
'namenode_heap_usage_percent': random.uniform(40, 85),
'datanode_count': random.randint(5, 20),
'total_capacity_tb': random.uniform(100, 1000),
'used_capacity_tb': random.uniform(50, 800),
'block_count': random.randint(1000000, 10000000)
}
elif component == 'yarn':
metrics = {
'memory_utilization_percent': random.uniform(30, 95),
'cpu_utilization_percent': random.uniform(20, 80),
'active_nodes': random.randint(5, 50),
'running_applications': random.randint(10, 100),
'pending_applications': random.randint(0, 20)
}
elif component == 'hive':
metrics = {
'active_sessions': random.randint(5, 50),
'avg_query_time_seconds': random.uniform(30, 600),
'query_success_rate_percent': random.uniform(85, 99),
'metastore_connections': random.randint(10, 100)
}
else:
metrics = {'status': 'unknown_component'}
# 添加时间戳
metrics['timestamp'] = datetime.now().isoformat()
metrics['component'] = component
# 保存到历史记录
self.metrics_history.append(metrics)
# 检查告警
self._check_alerts(component, metrics)
return {
'status': 'success',
'component': component,
'metrics': metrics
}
def _check_alerts(self, component: str, metrics: Dict[str, Any]):
"""检查告警条件"""
if component not in self.thresholds:
return
component_thresholds = self.thresholds[component]
for metric_name, threshold in component_thresholds.items():
if metric_name in metrics:
value = metrics[metric_name]
if value > threshold:
alert = {
'alert_id': f"{component}_{metric_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
'component': component,
'metric': metric_name,
'value': value,
'threshold': threshold,
'severity': self._get_severity(metric_name, value, threshold),
'timestamp': datetime.now().isoformat(),
'message': f"{component} {metric_name} is {value}, exceeds threshold {threshold}"
}
self.alerts.append(alert)
def _get_severity(self, metric_name: str, value: float, threshold: float) -> str:
"""获取告警严重程度"""
ratio = value / threshold
if ratio > 1.2:
return 'critical'
elif ratio > 1.1:
return 'warning'
else:
return 'info'
def get_alerts(self, severity: Optional[str] = None, limit: int = 50) -> Dict[str, Any]:
"""获取告警列表"""
filtered_alerts = self.alerts
if severity:
filtered_alerts = [a for a in self.alerts if a['severity'] == severity]
# 按时间倒序排列
filtered_alerts.sort(key=lambda x: x['timestamp'], reverse=True)
return {
'status': 'success',
'total_alerts': len(filtered_alerts),
'alerts': filtered_alerts[:limit]
}
def get_health_summary(self) -> Dict[str, Any]:
"""获取健康状况摘要"""
# 统计各组件状态
component_status = {}
recent_metrics = {}
# 获取最近的指标
for metrics in reversed(self.metrics_history[-50:]):
component = metrics['component']
if component not in recent_metrics:
recent_metrics[component] = metrics
# 评估组件健康状态
for component, metrics in recent_metrics.items():
health_score = self._calculate_health_score(component, metrics)
component_status[component] = {
'health_score': health_score,
'status': self._get_health_status(health_score),
'last_update': metrics['timestamp']
}
# 统计告警
alert_summary = {
'critical': len([a for a in self.alerts if a['severity'] == 'critical']),
'warning': len([a for a in self.alerts if a['severity'] == 'warning']),
'info': len([a for a in self.alerts if a['severity'] == 'info'])
}
return {
'status': 'success',
'overall_health': self._calculate_overall_health(component_status),
'component_status': component_status,
'alert_summary': alert_summary,
'total_components': len(component_status)
}
def _calculate_health_score(self, component: str, metrics: Dict[str, Any]) -> float:
"""计算健康分数 (0-100)"""
if component not in self.thresholds:
return 100.0
score = 100.0
thresholds = self.thresholds[component]
for metric_name, threshold in thresholds.items():
if metric_name in metrics:
value = metrics[metric_name]
if value > threshold:
# 根据超出程度扣分
ratio = value / threshold
penalty = min(50, (ratio - 1) * 100)
score -= penalty
return max(0.0, score)
def _get_health_status(self, health_score: float) -> str:
"""根据健康分数获取状态"""
if health_score >= 90:
return 'excellent'
elif health_score >= 70:
return 'good'
elif health_score >= 50:
return 'warning'
else:
return 'critical'
def _calculate_overall_health(self, component_status: Dict[str, Dict[str, Any]]) -> str:
"""计算整体健康状态"""
if not component_status:
return 'unknown'
avg_score = sum(status['health_score'] for status in component_status.values()) / len(component_status)
return self._get_health_status(avg_score)
# 使用示例
if __name__ == "__main__":
# 创建生态系统集成器
integrator = HadoopEcosystemIntegrator()
print("=== Hadoop生态系统集成示例 ===\n")
# 注册组件
print("=== 注册组件 ===")
components = [
ComponentIntegration(
component_name="kafka",
role=ComponentRole.DATA_INGESTION,
dependencies=[],
configuration={"brokers": 3, "partitions": 12}
),
ComponentIntegration(
component_name="storm",
role=ComponentRole.DATA_PROCESSING,
dependencies=["kafka"],
configuration={"workers": 4, "executors": 16}
),
ComponentIntegration(
component_name="hbase",
role=ComponentRole.DATA_STORAGE,
dependencies=["hdfs"],
configuration={"regions": 100, "replication": 3}
)
]
for component in components:
result = integrator.register_component(component)
print(f"注册 {component.component_name}: {result['status']}")
# 创建实时分析管道
print("\n=== 创建实时分析管道 ===")
pipeline_config = {
'pipeline_id': 'real-time-analytics-001',
'name': 'Real-time User Behavior Analytics',
'integration_type': 'real_time_analytics',
'schedule': '0 */6 * * *',
'sla_minutes': 30
}
pipeline_result = integrator.create_pipeline(pipeline_config)
print(f"管道创建结果: {pipeline_result}")
if pipeline_result['status'] == 'success':
pipeline_id = pipeline_result['pipeline_id']
# 验证管道
print("\n=== 验证管道配置 ===")
validation_result = integrator.validate_pipeline(pipeline_id)
print(f"验证结果: {validation_result['is_valid']}")
if validation_result['validation_results']:
for issue in validation_result['validation_results']:
print(f" 问题: {issue}")
# 获取管道状态
print("\n=== 管道状态 ===")
status_result = integrator.get_pipeline_status(pipeline_id)
if status_result['status'] == 'success':
print(f"管道ID: {status_result['pipeline_id']}")
print(f"名称: {status_result['name']}")
print(f"集成类型: {status_result['integration_type']}")
print(f"组件数量: {status_result['components_count']}")
print("组件健康状态:")
for health in status_result['component_health']:
print(f" - {health['component']} ({health['role']}): {health['status']}")
# 获取集成建议
print("\n=== 集成建议 ===")
requirements = {
'data_volume': 'large',
'latency': 'real_time',
'data_types': ['streaming', 'structured']
}
recommendations = integrator.get_integration_recommendations(requirements)
if recommendations['status'] == 'success':
print(f"基于需求的建议 ({recommendations['total_recommendations']}个):")
for rec in recommendations['recommendations']:
print(f" 架构: {rec['architecture']}")
print(f" 组件: {', '.join(rec['components'])}")
print(f" 描述: {rec['description']}\n")
# 性能优化分析
print("=== 性能优化分析 ===")
optimizer = PerformanceOptimizer()
# 模拟HDFS性能指标
hdfs_metrics = {
'avg_file_size_mb': 32, # 小文件问题
'disk_usage_percent': 85, # 磁盘使用率高
'network_utilization_percent': 45
}
hdfs_analysis = optimizer.analyze_performance('hdfs', hdfs_metrics)
if hdfs_analysis['status'] == 'success':
print(f"HDFS性能分析 ({hdfs_analysis['total_recommendations']}个建议):")
for rec in hdfs_analysis['recommendations']:
print(f" 类型: {rec['type']} (优先级: {rec['priority']})")
print(f" 描述: {rec['description']}")
print(f" 建议: {rec['action']}\n")
# 监控示例
print("=== 系统监控 ===")
monitor = EcosystemMonitor()
# 收集各组件指标
components_to_monitor = ['hdfs', 'yarn', 'hive']
for component in components_to_monitor:
metrics_result = monitor.collect_metrics(component)
if metrics_result['status'] == 'success':
metrics = metrics_result['metrics']
print(f"{component.upper()}指标:")
for key, value in metrics.items():
if key not in ['timestamp', 'component']:
if isinstance(value, float):
print(f" {key}: {value:.2f}")
else:
print(f" {key}: {value}")
print()
# 获取告警
print("=== 告警信息 ===")
alerts_result = monitor.get_alerts(limit=5)
if alerts_result['status'] == 'success':
print(f"总告警数: {alerts_result['total_alerts']}")
for alert in alerts_result['alerts']:
print(f" [{alert['severity'].upper()}] {alert['message']}")
# 获取健康摘要
print("\n=== 健康状况摘要 ===")
health_summary = monitor.get_health_summary()
if health_summary['status'] == 'success':
print(f"整体健康状态: {health_summary['overall_health']}")
print(f"监控组件数: {health_summary['total_components']}")
print("组件状态:")
for component, status in health_summary['component_status'].items():
print(f" {component}: {status['status']} (分数: {status['health_score']:.1f})")
alert_summary = health_summary['alert_summary']
print(f"告警统计: 严重 {alert_summary['critical']}, 警告 {alert_summary['warning']}, 信息 {alert_summary['info']}")
## 11. 总结与展望
### 11.1 Hadoop生态系统总结
Hadoop生态系统是一个庞大而复杂的大数据处理平台，包含了从数据存储、处理、查询到监控、安全等各个方面的组件。通过本教程的学习，我们深入了解了:
1. **核心存储组件**: HDFS提供分布式文件存储，HBase提供NoSQL数据库功能
2. **数据处理组件**: MapReduce、Spark、Storm等提供批处理和流处理能力
3. **数据查询组件**: Hive、Impala、Drill等提供SQL查询接口
4. **数据流组件**: Kafka、Pulsar提供消息队列和流处理平台
5. **监控管理组件**: Ambari、Cloudera Manager提供集群管理和监控
6. **工作流调度组件**: Airflow、Oozie提供任务调度和工作流管理
7. **数据传输组件**: Sqoop、Flume、NiFi提供数据导入导出和流式传输
8. **安全组件**: Ranger、Knox提供权限管理和安全网关
### 11.2 架构设计原则
在设计Hadoop生态系统架构时，应遵循以下原则:
1. **可扩展性**: 系统应能够水平扩展以处理不断增长的数据量
2. **容错性**: 系统应能够处理硬件故障和网络问题
3. **性能优化**: 通过合理的配置和优化提高系统性能
4. **安全性**: 实施适当的安全措施保护数据和系统
5. **可维护性**: 系统应易于监控、管理和维护
6. **成本效益**: 在满足需求的前提下控制成本
### 11.3 技术发展趋势
1. **云原生化**: 越来越多的Hadoop组件支持容器化部署和云原生架构
2. **实时处理**: 流处理技术的发展使得实时数据处理成为主流
3. **机器学习集成**: 大数据平台与机器学习框架的深度集成
4. **自动化运维**: 智能化的监控、调优和故障处理
5. **多云支持**: 支持跨云平台的数据处理和管理
### 11.4 学习建议
1. **实践为主**: 通过搭建实际环境来学习和理解各组件
2. **循序渐进**: 从核心组件开始，逐步学习扩展组件
3. **关注社区**: 跟踪开源社区的发展和最新技术
4. **项目实战**: 通过实际项目来应用所学知识
5. **持续学习**: 大数据技术发展迅速，需要持续学习新技术
### 11.5 未来展望
Hadoop生态系统将继续演进，主要发展方向包括:
1. **更好的用户体验**: 简化部署、配置和使用流程
2. **更强的性能**: 通过新的算法和优化技术提高处理性能
3. **更广的应用场景**: 支持更多类型的数据和应用场景
4. **更深的集成**: 与其他技术栈的更深度集成
5. **更智能的管理**: 基于AI的自动化管理和优化
通过掌握Hadoop生态系统，您将能够构建强大的大数据处理平台，为企业的数字化转型提供技术支撑。
---
**本教程到此结束。希望通过学习本教程，您能够深入理解Hadoop生态系统的各个组件，并能够在实际项目中应用这些知识。**
## 9. 安全组件
### 9.1 Apache Ranger详解
Apache Ranger是一个用于在Hadoop平台上启用、监控和管理全面数据安全的框架。
```python
from typing import Dict, List, Any, Optional, Set
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import json
import uuid
class PolicyType(Enum):
"""策略类型"""
ACCESS = "access"
MASKING = "masking"
ROW_FILTER = "row_filter"
TAG_BASED = "tag_based"
class PermissionType(Enum):
"""权限类型"""
READ = "read"
WRITE = "write"
CREATE = "create"
DELETE = "delete"
ADMIN = "admin"
SELECT = "select"
UPDATE = "update"
DROP = "drop"
ALTER = "alter"
INDEX = "index"
LOCK = "lock"
ALL = "all"
class ServiceType(Enum):
"""服务类型"""
HDFS = "hdfs"
HIVE = "hive"
HBASE = "hbase"
YARN = "yarn"
KAFKA = "kafka"
STORM = "storm"
KNOX = "knox"
SOLR = "solr"
ATLAS = "atlas"
class AuditAction(Enum):
"""审计动作"""
ACCESS = "access"
CREATE = "create"
UPDATE = "update"
DELETE = "delete"
LOGIN = "login"
LOGOUT = "logout"
POLICY_CHANGE = "policy_change"
ADMIN_ACTION = "admin_action"
class AccessResult(Enum):
"""访问结果"""
ALLOWED = "allowed"
DENIED = "denied"
NOT_DETERMINED = "not_determined"
@dataclass
class RangerResource:
"""Ranger资源"""
service_type: ServiceType
resource_type: str # database, table, column, path等
resource_name: str
resource_path: Optional[str] = None
is_recursive: bool = False
is_excludes: bool = False
@dataclass
class PolicyItem:
"""策略项"""
users: List[str] = field(default_factory=list)
groups: List[str] = field(default_factory=list)
roles: List[str] = field(default_factory=list)
permissions: List[PermissionType] = field(default_factory=list)
delegate_admin: bool = False
conditions: Dict[str, Any] = field(default_factory=dict)
@dataclass
class RangerPolicy:
"""Ranger策略"""
id: str
name: str
service_name: str
service_type: ServiceType
policy_type: PolicyType
description: str
is_enabled: bool
is_audit_enabled: bool
resources: Dict[str, RangerResource]
policy_items: List[PolicyItem]
deny_policy_items: List[PolicyItem] = field(default_factory=list)
allow_exceptions: List[PolicyItem] = field(default_factory=list)
deny_exceptions: List[PolicyItem] = field(default_factory=list)
created_by: str = "admin"
updated_by: str = "admin"
create_time: datetime = field(default_factory=datetime.now)
update_time: datetime = field(default_factory=datetime.now)
version: int = 1
@dataclass
class AuditEvent:
"""审计事件"""
id: str
event_time: datetime
user: str
service_name: str
service_type: ServiceType
resource_path: str
resource_type: str
action: AuditAction
access_type: str
result: AccessResult
policy_id: Optional[str] = None
client_ip: Optional[str] = None
session_id: Optional[str] = None
request_data: Optional[str] = None
additional_info: Dict[str, Any] = field(default_factory=dict)
@dataclass
class RangerService:
"""Ranger服务"""
id: str
name: str
type: ServiceType
description: str
is_enabled: bool
configs: Dict[str, Any]
created_by: str = "admin"
updated_by: str = "admin"
create_time: datetime = field(default_factory=datetime.now)
update_time: datetime = field(default_factory=datetime.now)
version: int = 1
@dataclass
class RangerUser:
"""Ranger用户"""
id: str
name: str
first_name: str
last_name: str
email_address: str
password: str
user_source: int = 0 # 0: Internal, 1: External
is_visible: int = 1
user_role_list: List[str] = field(default_factory=list)
group_id_list: List[str] = field(default_factory=list)
status: int = 1 # 1: Active, 0: Inactive
created_by: str = "admin"
updated_by: str = "admin"
create_time: datetime = field(default_factory=datetime.now)
update_time: datetime = field(default_factory=datetime.now)
@dataclass
class RangerGroup:
"""Ranger组"""
id: str
name: str
description: str
group_type: int = 1 # 1: Internal, 0: External
group_source: int = 0
is_visible: int = 1
created_by: str = "admin"
updated_by: str = "admin"
create_time: datetime = field(default_factory=datetime.now)
update_time: datetime = field(default_factory=datetime.now)
class RangerAdmin:
"""
Apache Ranger管理服务器
提供策略管理、用户管理、审计等功能
"""
def __init__(self, admin_url: str = "http://localhost:6080"):
self.admin_url = admin_url
self.services: Dict[str, RangerService] = {}
self.policies: Dict[str, RangerPolicy] = {}
self.users: Dict[str, RangerUser] = {}
self.groups: Dict[str, RangerGroup] = {}
self.audit_events: List[AuditEvent] = []
self.is_running = True
self.version = "2.4.0"
# 初始化默认数据
self._initialize_default_data()
def _initialize_default_data(self):
"""初始化默认数据"""
# 创建默认用户
admin_user = RangerUser(
id="user_1",
name="admin",
first_name="Admin",
last_name="User",
email_address="admin@example.com",
password="admin123",
user_role_list=["ROLE_SYS_ADMIN"]
)
self.users[admin_user.id] = admin_user
# 创建默认组
admin_group = RangerGroup(
id="group_1",
name="admin",
description="Administrator group"
)
self.groups[admin_group.id] = admin_group
# 创建默认服务
hdfs_service = RangerService(
id="service_1",
name="hadoop-hdfs",
type=ServiceType.HDFS,
description="HDFS Service",
is_enabled=True,
configs={
"username": "hdfs",
"password": "hdfs",
"fs.default.name": "hdfs://localhost:9000",
"hadoop.security.authorization": "true"
}
)
self.services[hdfs_service.id] = hdfs_service
def create_service(self, name: str, service_type: ServiceType,
description: str, configs: Dict[str, Any]) -> Dict[str, Any]:
"""
创建服务
Args:
name: 服务名称
service_type: 服务类型
description: 描述
configs: 配置
Returns:
Dict[str, Any]: 创建结果
"""
try:
service_id = f"service_{len(self.services) + 1}"
service = RangerService(
id=service_id,
name=name,
type=service_type,
description=description,
is_enabled=True,
configs=configs
)
self.services[service_id] = service
# 记录审计事件
self._log_audit_event(
user="admin",
service_name=name,
service_type=service_type,
resource_path=f"/service/{name}",
resource_type="service",
action=AuditAction.CREATE,
access_type="create",
result=AccessResult.ALLOWED
)
return {
'status': 'success',
'service_id': service_id,
'message': f'Service {name} created successfully'
}
except Exception as e:
return {
'status': 'error',
'message': f'Failed to create service: {str(e)}'
}
def create_policy(self, policy_name: str, service_name: str,
policy_type: PolicyType, resources: Dict[str, Any],
policy_items: List[Dict[str, Any]],
description: str = "") -> Dict[str, Any]:
"""
创建策略
Args:
policy_name: 策略名称
service_name: 服务名称
policy_type: 策略类型
resources: 资源定义
policy_items: 策略项
description: 描述
Returns:
Dict[str, Any]: 创建结果
"""
try:
# 查找服务
service = None
for svc in self.services.values():
if svc.name == service_name:
service = svc
break
if not service:
return {
'status': 'error',
'message': f'Service {service_name} not found'
}
policy_id = f"policy_{len(self.policies) + 1}"
# 转换资源
ranger_resources = {}
for res_type, res_values in resources.items():
if isinstance(res_values, str):
res_values = [res_values]
ranger_resources[res_type] = RangerResource(
service_type=service.type,
resource_type=res_type,
resource_name=",".join(res_values) if isinstance(res_values, list) else res_values
)
# 转换策略项
ranger_policy_items = []
for item in policy_items:
policy_item = PolicyItem(
users=item.get('users', []),
groups=item.get('groups', []),
roles=item.get('roles', []),
permissions=[PermissionType(p) for p in item.get('permissions', [])],
delegate_admin=item.get('delegate_admin', False)
)
ranger_policy_items.append(policy_item)
policy = RangerPolicy(
id=policy_id,
name=policy_name,
service_name=service_name,
service_type=service.type,
policy_type=policy_type,
description=description,
is_enabled=True,
is_audit_enabled=True,
resources=ranger_resources,
policy_items=ranger_policy_items
)
self.policies[policy_id] = policy
# 记录审计事件
self._log_audit_event(
user="admin",
service_name=service_name,
service_type=service.type,
resource_path=f"/policy/{policy_name}",
resource_type="policy",
action=AuditAction.CREATE,
access_type="create",
result=AccessResult.ALLOWED,
policy_id=policy_id
)
return {
'status': 'success',
'policy_id': policy_id,
'message': f'Policy {policy_name} created successfully'
}
except Exception as e:
return {
'status': 'error',
'message': f'Failed to create policy: {str(e)}'
}
def check_access(self, user: str, resource_path: str,
access_type: str, service_name: str) -> Dict[str, Any]:
"""
检查访问权限
Args:
user: 用户名
resource_path: 资源路径
access_type: 访问类型
service_name: 服务名称
Returns:
Dict[str, Any]: 访问检查结果
"""
try:
# 查找服务
service = None
for svc in self.services.values():
if svc.name == service_name:
service = svc
break
if not service:
return {
'status': 'error',
'message': f'Service {service_name} not found'
}
# 检查策略
access_result = AccessResult.DENIED
matched_policy = None
for policy in self.policies.values():
if (policy.service_name == service_name and
policy.is_enabled and
self._matches_resource(policy, resource_path)):
# 检查策略项
for item in policy.policy_items:
if (user in item.users and
any(perm.value == access_type or perm == PermissionType.ALL
for perm in item.permissions)):
access_result = AccessResult.ALLOWED
matched_policy = policy
break
if access_result == AccessResult.ALLOWED:
break
# 记录审计事件
self._log_audit_event(
user=user,
service_name=service_name,
service_type=service.type,
resource_path=resource_path,
resource_type="file",
action=AuditAction.ACCESS,
access_type=access_type,
result=access_result,
policy_id=matched_policy.id if matched_policy else None
)
return {
'status': 'success',
'access_result': access_result.value,
'policy_id': matched_policy.id if matched_policy else None,
'policy_name': matched_policy.name if matched_policy else None
}
except Exception as e:
return {
'status': 'error',
'message': f'Failed to check access: {str(e)}'
}
def _matches_resource(self, policy: RangerPolicy, resource_path: str) -> bool:
"""
检查资源是否匹配策略
Args:
policy: 策略
resource_path: 资源路径
Returns:
bool: 是否匹配
"""
# 简化的资源匹配逻辑
for resource in policy.resources.values():
if resource.resource_name in resource_path or resource_path.startswith(resource.resource_name):
return True
return False
def _log_audit_event(self, user: str, service_name: str, service_type: ServiceType,
resource_path: str, resource_type: str, action: AuditAction,
access_type: str, result: AccessResult, policy_id: Optional[str] = None):
"""
记录审计事件
Args:
user: 用户
service_name: 服务名称
service_type: 服务类型
resource_path: 资源路径
resource_type: 资源类型
action: 动作
access_type: 访问类型
result: 结果
policy_id: 策略ID
"""
event = AuditEvent(
id=f"audit_{len(self.audit_events) + 1}",
event_time=datetime.now(),
user=user,
service_name=service_name,
service_type=service_type,
resource_path=resource_path,
resource_type=resource_type,
action=action,
access_type=access_type,
result=result,
policy_id=policy_id,
client_ip="192.168.1.100",
session_id=f"session_{uuid.uuid4().hex[:8]}"
)
self.audit_events.append(event)
# 保持最近1000条审计记录
if len(self.audit_events) > 1000:
self.audit_events = self.audit_events[-1000:]
def get_policies(self, service_name: Optional[str] = None) -> Dict[str, Any]:
"""
获取策略列表
Args:
service_name: 服务名称（可选）
Returns:
Dict[str, Any]: 策略列表
"""
policies_info = []
for policy in self.policies.values():
if service_name is None or policy.service_name == service_name:
policies_info.append({
'id': policy.id,
'name': policy.name,
'service_name': policy.service_name,
'service_type': policy.service_type.value,
'policy_type': policy.policy_type.value,
'description': policy.description,
'is_enabled': policy.is_enabled,
'is_audit_enabled': policy.is_audit_enabled,
'created_by': policy.created_by,
'create_time': policy.create_time.isoformat(),
'version': policy.version
})
return {
'status': 'success',
'policies': policies_info,
'total': len(policies_info)
}
def get_audit_events(self, user: Optional[str] = None,
service_name: Optional[str] = None,
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
limit: int = 100) -> Dict[str, Any]:
"""
获取审计事件
Args:
user: 用户名（可选）
service_name: 服务名称（可选）
start_time: 开始时间（可选）
end_time: 结束时间（可选）
limit: 限制数量
Returns:
Dict[str, Any]: 审计事件列表
"""
filtered_events = []
for event in self.audit_events:
# 应用过滤条件
if user and event.user != user:
continue
if service_name and event.service_name != service_name:
continue
if start_time and event.event_time < start_time:
continue
if end_time and event.event_time > end_time:
continue
filtered_events.append({
'id': event.id,
'event_time': event.event_time.isoformat(),
'user': event.user,
'service_name': event.service_name,
'service_type': event.service_type.value,
'resource_path': event.resource_path,
'resource_type': event.resource_type,
'action': event.action.value,
'access_type': event.access_type,
'result': event.result.value,
'policy_id': event.policy_id,
'client_ip': event.client_ip,
'session_id': event.session_id
})
# 按时间倒序排列并限制数量
filtered_events.sort(key=lambda x: x['event_time'], reverse=True)
filtered_events = filtered_events[:limit]
return {
'status': 'success',
'events': filtered_events,
'total': len(filtered_events)
}
def get_services(self) -> Dict[str, Any]:
"""
获取服务列表
Returns:
Dict[str, Any]: 服务列表
"""
services_info = []
for service in self.services.values():
services_info.append({
'id': service.id,
'name': service.name,
'type': service.type.value,
'description': service.description,
'is_enabled': service.is_enabled,
'created_by': service.created_by,
'create_time': service.create_time.isoformat(),
'version': service.version
})
return {
'status': 'success',
'services': services_info,
'total': len(services_info)
}
def get_admin_status(self) -> Dict[str, Any]:
"""
获取管理服务器状态
Returns:
Dict[str, Any]: 服务器状态
"""
return {
'admin_url': self.admin_url,
'is_running': self.is_running,
'version': self.version,
'stats': {
'total_services': len(self.services),
'total_policies': len(self.policies),
'total_users': len(self.users),
'total_groups': len(self.groups),
'total_audit_events': len(self.audit_events),
'enabled_services': len([s for s in self.services.values() if s.is_enabled]),
'enabled_policies': len([p for p in self.policies.values() if p.is_enabled])
},
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Ranger管理服务器
ranger = RangerAdmin()
print("=== Apache Ranger安全管理示例 ===")
# 创建Hive服务
print("\n=== 创建Hive服务 ===")
hive_service_result = ranger.create_service(
name="hadoop-hive",
service_type=ServiceType.HIVE,
description="Hive Service for data warehouse",
configs={
"username": "hive",
"password": "hive123",
"jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver",
"jdbc.url": "jdbc:hive2://localhost:10000"
}
)
print(f"Hive服务创建结果: {hive_service_result}")
# 创建HDFS访问策略
print("\n=== 创建HDFS访问策略 ===")
hdfs_policy_result = ranger.create_policy(
policy_name="hdfs-data-access",
service_name="hadoop-hdfs",
policy_type=PolicyType.ACCESS,
resources={
"path": ["/user/data/*"]
},
policy_items=[
{
"users": ["datauser", "analyst"],
"groups": ["data-team"],
"permissions": ["read", "write"]
}
],
description="Allow data team to access /user/data directory"
)
print(f"HDFS策略创建结果: {hdfs_policy_result}")
# 创建Hive数据库策略
print("\n=== 创建Hive数据库策略 ===")
hive_policy_result = ranger.create_policy(
policy_name="hive-sales-db-access",
service_name="hadoop-hive",
policy_type=PolicyType.ACCESS,
resources={
"database": ["sales"],
"table": ["*"],
"column": ["*"]
},
policy_items=[
{
"users": ["analyst"],
"groups": ["analytics-team"],
"permissions": ["select"]
},
{
"users": ["dataadmin"],
"permissions": ["all"]
}
],
description="Access control for sales database"
)
print(f"Hive策略创建结果: {hive_policy_result}")
# 检查访问权限
print("\n=== 检查访问权限 ===")
# 检查HDFS访问
hdfs_access_check = ranger.check_access(
user="datauser",
resource_path="/user/data/sales.csv",
access_type="read",
service_name="hadoop-hdfs"
)
print(f"datauser访问/user/data/sales.csv (read): {hdfs_access_check}")
# 检查Hive访问
hive_access_check = ranger.check_access(
user="analyst",
resource_path="sales.customers",
access_type="select",
service_name="hadoop-hive"
)
print(f"analyst访问sales.customers (select): {hive_access_check}")
# 检查未授权访问
unauthorized_check = ranger.check_access(
user="guest",
resource_path="/user/data/confidential.csv",
access_type="read",
service_name="hadoop-hdfs"
)
print(f"guest访问/user/data/confidential.csv (read): {unauthorized_check}")
# 获取策略列表
print("\n=== 策略列表 ===")
policies = ranger.get_policies()
if policies['status'] == 'success':
print(f"总策略数: {policies['total']}")
for policy in policies['policies']:
print(f" - {policy['name']} ({policy['service_name']}) - {policy['policy_type']}")
# 获取审计事件
print("\n=== 审计事件 ===")
audit_events = ranger.get_audit_events(limit=10)
if audit_events['status'] == 'success':
print(f"审计事件数: {audit_events['total']}")
for event in audit_events['events'][:5]: # 显示前5个事件
print(f" - {event['event_time']}: {event['user']} {event['action']} {event['resource_path']} -> {event['result']}")
# 获取服务列表
print("\n=== 服务列表 ===")
services = ranger.get_services()
if services['status'] == 'success':
print(f"总服务数: {services['total']}")
for service in services['services']:
print(f" - {service['name']} ({service['type']}) - {'启用' if service['is_enabled'] else '禁用'}")
# 获取管理服务器状态
print("\n=== 管理服务器状态 ===")
status = ranger.get_admin_status()
print(f"服务器URL: {status['admin_url']}")
print(f"运行状态: {status['is_running']}")
print(f"版本: {status['version']}")
print("统计信息:")
for key, value in status['stats'].items():
print(f" {key}: {value}")
9.2 Apache Knox详解
Apache Knox是一个为Hadoop集群提供安全访问的网关服务。
from typing import Dict, List, Any, Optional
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import base64
import hashlib
import jwt
import uuid
class AuthenticationType(Enum):
"""认证类型"""
BASIC = "basic"
KERBEROS = "kerberos"
JWT = "jwt"
LDAP = "ldap"
PAM = "pam"
class ServiceRole(Enum):
"""服务角色"""
NAMENODE = "NAMENODE"
DATANODE = "DATANODE"
RESOURCEMANAGER = "RESOURCEMANAGER"
NODEMANAGER = "NODEMANAGER"
HIVESERVER2 = "HIVESERVER2"
WEBHCAT = "WEBHCAT"
OOZIE = "OOZIE"
WEBHBASE = "WEBHBASE"
STORM = "STORM"
KAFKA = "KAFKA"
class TopologyStatus(Enum):
"""拓扑状态"""
ACTIVE = "active"
INACTIVE = "inactive"
DEPLOYING = "deploying"
ERROR = "error"
@dataclass
class KnoxService:
"""Knox服务定义"""
name: str
role: ServiceRole
url: str
version: str = "1.0.0"
params: Dict[str, Any] = field(default_factory=dict)
@dataclass
class KnoxProvider:
"""Knox提供者"""
name: str
enabled: bool
role: str # authentication, authorization, identity-assertion等
params: Dict[str, Any] = field(default_factory=dict)
@dataclass
class KnoxTopology:
"""Knox拓扑"""
name: str
description: str
services: List[KnoxService]
providers: List[KnoxProvider]
status: TopologyStatus = TopologyStatus.INACTIVE
created_time: datetime = field(default_factory=datetime.now)
updated_time: datetime = field(default_factory=datetime.now)
version: str = "1.0.0"
@dataclass
class KnoxUser:
"""Knox用户"""
username: str
password_hash: str
roles: List[str] = field(default_factory=list)
groups: List[str] = field(default_factory=list)
is_active: bool = True
created_time: datetime = field(default_factory=datetime.now)
last_login: Optional[datetime] = None
@dataclass
class KnoxSession:
"""Knox会话"""
session_id: str
username: str
client_ip: str
user_agent: str
created_time: datetime
last_access_time: datetime
expires_at: datetime
is_active: bool = True
jwt_token: Optional[str] = None
@dataclass
class AccessLog:
"""访问日志"""
timestamp: datetime
client_ip: str
username: str
method: str
url: str
status_code: int
response_size: int
user_agent: str
session_id: str
topology_name: str
service_name: str
response_time_ms: int
class KnoxGateway:
"""
Apache Knox网关
提供Hadoop集群的安全访问入口
"""
def __init__(self, gateway_host: str = "localhost", gateway_port: int = 8443):
self.gateway_host = gateway_host
self.gateway_port = gateway_port
self.gateway_url = f"https://{gateway_host}:{gateway_port}"
self.topologies: Dict[str, KnoxTopology] = {}
self.users: Dict[str, KnoxUser] = {}
self.sessions: Dict[str, KnoxSession] = {}
self.access_logs: List[AccessLog] = []
self.is_running = True
self.version = "1.6.1"
self.jwt_secret = "knox-secret-key"
# 初始化默认数据
self._initialize_default_data()
def _initialize_default_data(self):
"""初始化默认数据"""
# 创建默认用户
admin_user = KnoxUser(
username="admin",
password_hash=self._hash_password("admin123"),
roles=["admin"],
groups=["administrators"]
)
self.users["admin"] = admin_user
guest_user = KnoxUser(
username="guest",
password_hash=self._hash_password("guest123"),
roles=["user"],
groups=["users"]
)
self.users["guest"] = guest_user
# 创建默认拓扑
self._create_default_topology()
def _create_default_topology(self):
"""创建默认拓扑"""
# 定义服务
services = [
KnoxService(
name="namenode",
role=ServiceRole.NAMENODE,
url="hdfs://localhost:9000"
),
KnoxService(
name="resourcemanager",
role=ServiceRole.RESOURCEMANAGER,
url="http://localhost:8088"
),
KnoxService(
name="hive",
role=ServiceRole.HIVESERVER2,
url="http://localhost:10001"
)
]
# 定义提供者
providers = [
KnoxProvider(
name="ShiroProvider",
enabled=True,
role="authentication",
params={
"sessionTimeout": "30",
"main.ldapRealm": "org.apache.knox.gateway.shirorealm.KnoxLdapRealm",
"main.ldapContextFactory": "org.apache.knox.gateway.shirorealm.KnoxLdapContextFactory",
"main.ldapRealm.contextFactory": "$ldapContextFactory"
}
),
KnoxProvider(
name="AclsAuthz",
enabled=True,
role="authorization",
params={
"webhdfs.acl": "admin;*;*",
"webhcat.acl": "admin;*;*"
}
),
KnoxProvider(
name="Default",
enabled=True,
role="identity-assertion",
params={
"name": "Default"
}
)
]
default_topology = KnoxTopology(
name="sandbox",
description="Default sandbox topology",
services=services,
providers=providers,
status=TopologyStatus.ACTIVE
)
self.topologies["sandbox"] = default_topology
def _hash_password(self, password: str) -> str:
"""密码哈希"""
return hashlib.sha256(password.encode()).hexdigest()
def authenticate(self, username: str, password: str,
auth_type: AuthenticationType = AuthenticationType.BASIC) -> Dict[str, Any]:
"""
用户认证
Args:
username: 用户名
password: 密码
auth_type: 认证类型
Returns:
Dict[str, Any]: 认证结果
"""
try:
if username not in self.users:
return {
'status': 'error',
'message': 'User not found'
}
user = self.users[username]
if not user.is_active:
return {
'status': 'error',
'message': 'User account is disabled'
}
# 验证密码
if user.password_hash != self._hash_password(password):
return {
'status': 'error',
'message': 'Invalid credentials'
}
# 创建会话
session_id = str(uuid.uuid4())
expires_at = datetime.now() + timedelta(hours=8)
# 生成JWT令牌
jwt_payload = {
'username': username,
'roles': user.roles,
'groups': user.groups,
'session_id': session_id,
'exp': expires_at.timestamp(),
'iat': datetime.now().timestamp()
}
jwt_token = jwt.encode(jwt_payload, self.jwt_secret, algorithm='HS256')
session = KnoxSession(
session_id=session_id,
username=username,
client_ip="192.168.1.100",
user_agent="Knox-Client/1.0",
created_time=datetime.now(),
last_access_time=datetime.now(),
expires_at=expires_at,
jwt_token=jwt_token
)
self.sessions[session_id] = session
# 更新用户最后登录时间
user.last_login = datetime.now()
return {
'status': 'success',
'session_id': session_id,
'jwt_token': jwt_token,
'expires_at': expires_at.isoformat(),
'user_info': {
'username': username,
'roles': user.roles,
'groups': user.groups
}
}
except Exception as e:
return {
'status': 'error',
'message': f'Authentication failed: {str(e)}'
}
def create_topology(self, name: str, description: str,
services: List[Dict[str, Any]],
providers: List[Dict[str, Any]]) -> Dict[str, Any]:
"""
创建拓扑
Args:
name: 拓扑名称
description: 描述
services: 服务列表
providers: 提供者列表
Returns:
Dict[str, Any]: 创建结果
"""
try:
if name in self.topologies:
return {
'status': 'error',
'message': f'Topology {name} already exists'
}
# 转换服务
knox_services = []
for svc in services:
knox_service = KnoxService(
name=svc['name'],
role=ServiceRole(svc['role']),
url=svc['url'],
version=svc.get('version', '1.0.0'),
params=svc.get('params', {})
)
knox_services.append(knox_service)
# 转换提供者
knox_providers = []
for prov in providers:
knox_provider = KnoxProvider(
name=prov['name'],
enabled=prov.get('enabled', True),
role=prov['role'],
params=prov.get('params', {})
)
knox_providers.append(knox_provider)
topology = KnoxTopology(
name=name,
description=description,
services=knox_services,
providers=knox_providers,
status=TopologyStatus.DEPLOYING
)
self.topologies[name] = topology
# 模拟部署过程
import time
time.sleep(1)
topology.status = TopologyStatus.ACTIVE
return {
'status': 'success',
'topology_name': name,
'message': f'Topology {name} created and deployed successfully'
}
except Exception as e:
return {
'status': 'error',
'message': f'Failed to create topology: {str(e)}'
}
def proxy_request(self, topology_name: str, service_name: str,
path: str, method: str = "GET",
session_id: Optional[str] = None) -> Dict[str, Any]:
"""
代理请求到后端服务
Args:
topology_name: 拓扑名称
service_name: 服务名称
path: 请求路径
method: HTTP方法
session_id: 会话ID
Returns:
Dict[str, Any]: 代理结果
"""
try:
# 检查拓扑
if topology_name not in self.topologies:
return {
'status': 'error',
'status_code': 404,
'message': f'Topology {topology_name} not found'
}
topology = self.topologies[topology_name]
if topology.status != TopologyStatus.ACTIVE:
return {
'status': 'error',
'status_code': 503,
'message': f'Topology {topology_name} is not active'
}
# 检查会话（如果提供）
username = "anonymous"
if session_id:
if session_id not in self.sessions:
return {
'status': 'error',
'status_code': 401,
'message': 'Invalid session'
}
session = self.sessions[session_id]
if not session.is_active or datetime.now() > session.expires_at:
return {
'status': 'error',
'status_code': 401,
'message': 'Session expired'
}
username = session.username
session.last_access_time = datetime.now()
# 查找服务
target_service = None
for service in topology.services:
if service.name == service_name:
target_service = service
break
if not target_service:
return {
'status': 'error',
'status_code': 404,
'message': f'Service {service_name} not found in topology {topology_name}'
}
# 模拟代理请求
start_time = datetime.now()
# 构建目标URL
target_url = f"{target_service.url}{path}"
# 模拟响应
if service_name == "namenode" and "webhdfs" in path:
response_data = {
"FileStatuses": {
"FileStatus": [
{
"accessTime": 1640995200000,
"blockSize": 134217728,
"group": "supergroup",
"length": 1024,
"modificationTime": 1640995200000,
"owner": "hdfs",
"pathSuffix": "data.txt",
"permission": "644",
"replication": 3,
"type": "FILE"
}
]
}
}
status_code = 200
elif service_name == "resourcemanager":
response_data = {
"clusterInfo": {
"id": 1640995200000,
"startedOn": 1640995200000,
"state": "STARTED",
"haState": "ACTIVE",
"resourceManagerVersion": "3.3.4",
"resourceManagerBuildVersion": "3.3.4",
"hadoopVersion": "3.3.4"
}
}
status_code = 200
else:
response_data = {"message": "Service response", "path": path}
status_code = 200
end_time = datetime.now()
response_time_ms = int((end_time - start_time).total_seconds() * 1000)
# 记录访问日志
access_log = AccessLog(
timestamp=start_time,
client_ip="192.168.1.100",
username=username,
method=method,
url=f"/gateway/{topology_name}/{service_name}{path}",
status_code=status_code,
response_size=len(str(response_data)),
user_agent="Knox-Client/1.0",
session_id=session_id or "anonymous",
topology_name=topology_name,
service_name=service_name,
response_time_ms=response_time_ms
)
self.access_logs.append(access_log)
# 保持最近1000条访问日志
if len(self.access_logs) > 1000:
self.access_logs = self.access_logs[-1000:]
return {
'status': 'success',
'status_code': status_code,
'target_url': target_url,
'response_data': response_data,
'response_time_ms': response_time_ms
}
except Exception as e:
return {
'status': 'error',
'status_code': 500,
'message': f'Proxy request failed: {str(e)}'
}
def get_topologies(self) -> Dict[str, Any]:
"""
获取拓扑列表
Returns:
Dict[str, Any]: 拓扑列表
"""
topologies_info = []
for topology in self.topologies.values():
topologies_info.append({
'name': topology.name,
'description': topology.description,
'status': topology.status.value,
'services_count': len(topology.services),
'providers_count': len(topology.providers),
'created_time': topology.created_time.isoformat(),
'updated_time': topology.updated_time.isoformat(),
'version': topology.version
})
return {
'status': 'success',
'topologies': topologies_info,
'total': len(topologies_info)
}
def get_access_logs(self, topology_name: Optional[str] = None,
username: Optional[str] = None,
limit: int = 100) -> Dict[str, Any]:
"""
获取访问日志
Args:
topology_name: 拓扑名称（可选）
username: 用户名（可选）
limit: 限制数量
Returns:
Dict[str, Any]: 访问日志列表
"""
filtered_logs = []
for log in self.access_logs:
if topology_name and log.topology_name != topology_name:
continue
if username and log.username != username:
continue
filtered_logs.append({
'timestamp': log.timestamp.isoformat(),
'client_ip': log.client_ip,
'username': log.username,
'method': log.method,
'url': log.url,
'status_code': log.status_code,
'response_size': log.response_size,
'user_agent': log.user_agent,
'session_id': log.session_id,
'topology_name': log.topology_name,
'service_name': log.service_name,
'response_time_ms': log.response_time_ms
})
# 按时间倒序排列并限制数量
filtered_logs.sort(key=lambda x: x['timestamp'], reverse=True)
filtered_logs = filtered_logs[:limit]
return {
'status': 'success',
'logs': filtered_logs,
'total': len(filtered_logs)
}
def get_gateway_status(self) -> Dict[str, Any]:
"""
获取网关状态
Returns:
Dict[str, Any]: 网关状态
"""
active_sessions = len([s for s in self.sessions.values()
if s.is_active and datetime.now() <= s.expires_at])
return {
'gateway_url': self.gateway_url,
'is_running': self.is_running,
'version': self.version,
'stats': {
'total_topologies': len(self.topologies),
'active_topologies': len([t for t in self.topologies.values()
if t.status == TopologyStatus.ACTIVE]),
'total_users': len(self.users),
'active_sessions': active_sessions,
'total_access_logs': len(self.access_logs)
},
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Knox网关
knox = KnoxGateway()
print("=== Apache Knox网关示例 ===")
# 用户认证
print("\n=== 用户认证 ===")
auth_result = knox.authenticate("admin", "admin123")
print(f"认证结果: {auth_result}")
if auth_result['status'] == 'success':
session_id = auth_result['session_id']
jwt_token = auth_result['jwt_token']
# 创建自定义拓扑
print("\n=== 创建自定义拓扑 ===")
topology_result = knox.create_topology(
name="production",
description="Production environment topology",
services=[
{
"name": "namenode",
"role": "NAMENODE",
"url": "hdfs://prod-namenode:9000"
},
{
"name": "resourcemanager",
"role": "RESOURCEMANAGER",
"url": "http://prod-rm:8088"
},
{
"name": "hive",
"role": "HIVESERVER2",
"url": "http://prod-hive:10001"
}
],
providers=[
{
"name": "ShiroProvider",
"role": "authentication",
"enabled": True,
"params": {
"sessionTimeout": "60"
}
},
{
"name": "AclsAuthz",
"role": "authorization",
"enabled": True,
"params": {
"webhdfs.acl": "admin;*;*"
}
}
]
)
print(f"拓扑创建结果: {topology_result}")
# 代理请求
print("\n=== 代理请求 ===")
# HDFS WebHDFS请求
hdfs_request = knox.proxy_request(
topology_name="sandbox",
service_name="namenode",
path="/webhdfs/v1/user?op=LISTSTATUS",
method="GET",
session_id=session_id
)
print(f"HDFS请求结果: {hdfs_request}")
# YARN ResourceManager请求
yarn_request = knox.proxy_request(
topology_name="sandbox",
service_name="resourcemanager",
path="/ws/v1/cluster/info",
method="GET",
session_id=session_id
)
print(f"YARN请求结果: {yarn_request}")
# 未授权请求
unauthorized_request = knox.proxy_request(
topology_name="sandbox",
service_name="namenode",
path="/webhdfs/v1/admin?op=LISTSTATUS",
method="GET"
)
print(f"未授权请求结果: {unauthorized_request}")
# 获取拓扑列表
print("\n=== 拓扑列表 ===")
topologies = knox.get_topologies()
if topologies['status'] == 'success':
print(f"总拓扑数: {topologies['total']}")
for topology in topologies['topologies']:
print(f" - {topology['name']}: {topology['description']} ({topology['status']})")
# 获取访问日志
print("\n=== 访问日志 ===")
access_logs = knox.get_access_logs(limit=5)
if access_logs['status'] == 'success':
print(f"访问日志数: {access_logs['total']}")
for log in access_logs['logs']:
print(f" - {log['timestamp']}: {log['username']} {log['method']} {log['url']} -> {log['status_code']} ({log['response_time_ms']}ms)")
# 获取网关状态
print("\n=== 网关状态 ===")
status = knox.get_gateway_status()
print(f"网关URL: {status['gateway_url']}")
print(f"运行状态: {status['is_running']}")
print(f"版本: {status['version']}")
print("统计信息:")
for key, value in status['stats'].items():
print(f" {key}: {value}")
print("资源池:")
for pool_name, pool_info in status['pools'].items():
print(f" {pool_name}: {pool_info['slots']} slots")
### 7.2 Apache Oozie详解
```python
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import xml.etree.ElementTree as ET
class WorkflowStatus(Enum):
"""工作流状态"""
PREP = "PREP" # 准备中
RUNNING = "RUNNING" # 运行中
SUCCEEDED = "SUCCEEDED" # 成功
FAILED = "FAILED" # 失败
KILLED = "KILLED" # 已终止
SUSPENDED = "SUSPENDED" # 暂停
class ActionStatus(Enum):
"""动作状态"""
PREP = "PREP" # 准备中
RUNNING = "RUNNING" # 运行中
OK = "OK" # 成功
ERROR = "ERROR" # 错误
KILLED = "KILLED" # 已终止
START_RETRY = "START_RETRY" # 开始重试
START_MANUAL = "START_MANUAL" # 手动开始
DONE = "DONE" # 完成
END_RETRY = "END_RETRY" # 结束重试
END_MANUAL = "END_MANUAL" # 手动结束
USER_RETRY = "USER_RETRY" # 用户重试
class CoordinatorStatus(Enum):
"""协调器状态"""
PREP = "PREP" # 准备中
RUNNING = "RUNNING" # 运行中
SUCCEEDED = "SUCCEEDED" # 成功
FAILED = "FAILED" # 失败
KILLED = "KILLED" # 已终止
SUSPENDED = "SUSPENDED" # 暂停
DONEWITHERROR = "DONEWITHERROR" # 完成但有错误
RUNNINGWITHERROR = "RUNNINGWITHERROR" # 运行中但有错误
class ActionType(Enum):
"""动作类型"""
MAP_REDUCE = "map-reduce"
PIG = "pig"
HIVE = "hive"
SQOOP = "sqoop"
SHELL = "shell"
JAVA = "java"
FS = "fs"
EMAIL = "email"
SSH = "ssh"
SUB_WORKFLOW = "sub-workflow"
SPARK = "spark"
DISTCP = "distcp"
class BundleStatus(Enum):
"""Bundle状态"""
PREP = "PREP" # 准备中
RUNNING = "RUNNING" # 运行中
SUCCEEDED = "SUCCEEDED" # 成功
FAILED = "FAILED" # 失败
KILLED = "KILLED" # 已终止
SUSPENDED = "SUSPENDED" # 暂停
DONEWITHERROR = "DONEWITHERROR" # 完成但有错误
RUNNINGWITHERROR = "RUNNINGWITHERROR" # 运行中但有错误
@dataclass
class WorkflowAction:
"""工作流动作"""
name: str
action_type: ActionType
status: ActionStatus = ActionStatus.PREP
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
transition: str = "" # 成功时的下一个动作
error_transition: str = "" # 失败时的下一个动作
retry_max: int = 3
retry_interval: int = 10 # 分钟
configuration: Dict[str, Any] = field(default_factory=dict)
external_id: str = ""
external_status: str = ""
external_child_ids: str = ""
error_code: str = ""
error_message: str = ""
data: str = ""
stats: str = ""
console_url: str = ""
user_retry_count: int = 0
user_retry_max: int = 3
user_retry_interval: int = 10
@dataclass
class WorkflowJob:
"""工作流作业"""
id: str
app_name: str
app_path: str
status: WorkflowStatus = WorkflowStatus.PREP
user: str = ""
group: str = ""
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
created_time: datetime = field(default_factory=datetime.now)
last_modified_time: datetime = field(default_factory=datetime.now)
run: int = 0
console_url: str = ""
actions: Dict[str, WorkflowAction] = field(default_factory=dict)
conf: str = ""
parent_id: str = ""
external_id: str = ""
log_token: str = ""
@dataclass
class CoordinatorAction:
"""协调器动作"""
id: str
job_id: str
status: ActionStatus = ActionStatus.PREP
external_id: str = ""
external_status: str = ""
nominal_time: Optional[datetime] = None
created_time: datetime = field(default_factory=datetime.now)
last_modified_time: datetime = field(default_factory=datetime.now)
missing_dependencies: str = ""
push_missing_dependencies: str = ""
timeout: int = 120 # 分钟
error_code: str = ""
error_message: str = ""
console_url: str = ""
@dataclass
class CoordinatorJob:
"""协调器作业"""
id: str
app_name: str
app_path: str
status: CoordinatorStatus = CoordinatorStatus.PREP
user: str = ""
group: str = ""
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
pause_time: Optional[datetime] = None
created_time: datetime = field(default_factory=datetime.now)
last_modified_time: datetime = field(default_factory=datetime.now)
next_materialized_time: Optional[datetime] = None
frequency: str = "" # 频率表达式
time_zone: str = "UTC"
concurrency: int = 1
execution: str = "FIFO" # FIFO, LIFO, LAST_ONLY
timeout: int = 120 # 分钟
actions: Dict[str, CoordinatorAction] = field(default_factory=dict)
conf: str = ""
mat_throttling: int = 12
@dataclass
class BundleJob:
"""Bundle作业"""
id: str
app_name: str
app_path: str
status: BundleStatus = BundleStatus.PREP
user: str = ""
group: str = ""
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
pause_time: Optional[datetime] = None
created_time: datetime = field(default_factory=datetime.now)
last_modified_time: datetime = field(default_factory=datetime.now)
kick_off_time: Optional[datetime] = None
time_zone: str = "UTC"
coordinators: List[str] = field(default_factory=list)
conf: str = ""
class OozieServer:
"""
Oozie工作流引擎服务器
"""
def __init__(self, server_url: str = "http://localhost:11000/oozie"):
self.server_url = server_url
self.workflow_jobs = {} # job_id -> WorkflowJob
self.coordinator_jobs = {} # job_id -> CoordinatorJob
self.bundle_jobs = {} # job_id -> BundleJob
# 服务器状态
self.is_running = True
self.build_version = "5.2.1"
self.system_mode = "NORMAL" # NORMAL, NOWEBSERVICE, SAFEMODE
self.java_version = "1.8.0_281"
self.os_name = "Linux"
self.os_version = "3.10.0"
# 配置
self.config = {
'oozie.service.WorkflowStoreService.workflow.callable.limit': 100,
'oozie.service.coord.normal.default.timeout': 120,
'oozie.service.coord.push.check.requeue.interval': 30000,
'oozie.service.CallableQueueService.queue.size': 10000,
'oozie.service.CallableQueueService.threads': 10,
'oozie.service.coord.materialization.window': 3600000,
'oozie.service.coord.lookup.trigger.window': 10800000,
'oozie.service.bundle.ForkedActionExecutor.buffer.size': 1000
}
# 线程锁
self.workflow_lock = threading.Lock()
self.coordinator_lock = threading.Lock()
self.bundle_lock = threading.Lock()
# 统计信息
self.stats = {
'total_workflows': 0,
'running_workflows': 0,
'succeeded_workflows': 0,
'failed_workflows': 0,
'total_coordinators': 0,
'running_coordinators': 0,
'total_bundles': 0,
'running_bundles': 0
}
# 初始化示例作业
self._create_example_jobs()
def _create_example_jobs(self):
"""创建示例作业"""
# 创建示例工作流
workflow_id = f"0000001-{datetime.now().strftime('%Y%m%d%H%M%S')}-oozie-W"
# 创建工作流动作
start_action = WorkflowAction(
name="start",
action_type=ActionType.FS,
status=ActionStatus.OK,
transition="data-processing"
)
data_processing_action = WorkflowAction(
name="data-processing",
action_type=ActionType.MAP_REDUCE,
status=ActionStatus.PREP,
transition="data-validation",
error_transition="fail",
configuration={
'mapred.job.queue.name': 'default',
'mapred.mapper.class': 'org.example.DataMapper',
'mapred.reducer.class': 'org.example.DataReducer',
'mapred.input.dir': '/user/input',
'mapred.output.dir': '/user/output'
}
)
data_validation_action = WorkflowAction(
name="data-validation",
action_type=ActionType.HIVE,
status=ActionStatus.PREP,
transition="end",
error_transition="fail",
configuration={
'hive.script': 'validate_data.hql',
'hive.params': 'INPUT=/user/output,OUTPUT=/user/validated'
}
)
end_action = WorkflowAction(
name="end",
action_type=ActionType.FS,
status=ActionStatus.PREP
)
fail_action = WorkflowAction(
name="fail",
action_type=ActionType.FS,
status=ActionStatus.PREP
)
workflow_job = WorkflowJob(
id=workflow_id,
app_name="data-processing-workflow",
app_path="/user/oozie/workflows/data-processing",
user="hadoop",
group="hadoop",
actions={
"start": start_action,
"data-processing": data_processing_action,
"data-validation": data_validation_action,
"end": end_action,
"fail": fail_action
},
conf='<configuration><property><name>jobTracker</name><value>localhost:8032</value></property></configuration>'
)
self.workflow_jobs[workflow_id] = workflow_job
self.stats['total_workflows'] += 1
# 创建示例协调器
coordinator_id = f"0000001-{datetime.now().strftime('%Y%m%d%H%M%S')}-oozie-C"
coordinator_job = CoordinatorJob(
id=coordinator_id,
app_name="daily-data-processing",
app_path="/user/oozie/coordinators/daily-processing",
user="hadoop",
group="hadoop",
start_time=datetime.now(),
end_time=datetime.now() + timedelta(days=30),
frequency="${coord:days(1)}",
concurrency=1,
execution="FIFO",
timeout=120
)
self.coordinator_jobs[coordinator_id] = coordinator_job
self.stats['total_coordinators'] += 1
# 创建示例Bundle
bundle_id = f"0000001-{datetime.now().strftime('%Y%m%d%H%M%S')}-oozie-B"
bundle_job = BundleJob(
id=bundle_id,
app_name="data-pipeline-bundle",
app_path="/user/oozie/bundles/data-pipeline",
user="hadoop",
group="hadoop",
start_time=datetime.now(),
end_time=datetime.now() + timedelta(days=90),
coordinators=[coordinator_id]
)
self.bundle_jobs[bundle_id] = bundle_job
self.stats['total_bundles'] += 1
def submit_workflow(self, app_path: str, config: Dict[str, Any],
user: str = "hadoop") -> Dict[str, Any]:
"""
提交工作流
Args:
app_path: 应用路径
config: 配置参数
user: 用户名
Returns:
Dict[str, Any]: 提交结果
"""
job_id = f"{len(self.workflow_jobs)+1:07d}-{datetime.now().strftime('%Y%m%d%H%M%S')}-oozie-W"
# 解析应用名称
app_name = app_path.split('/')[-1] if app_path else "workflow"
# 创建工作流作业
workflow_job = WorkflowJob(
id=job_id,
app_name=app_name,
app_path=app_path,
user=user,
group=user,
conf=json.dumps(config)
)
# 创建基本动作
start_action = WorkflowAction(
name="start",
action_type=ActionType.FS,
status=ActionStatus.OK,
transition="main-action"
)
main_action = WorkflowAction(
name="main-action",
action_type=ActionType.MAP_REDUCE,
status=ActionStatus.PREP,
transition="end",
error_transition="fail",
configuration=config
)
end_action = WorkflowAction(
name="end",
action_type=ActionType.FS,
status=ActionStatus.PREP
)
fail_action = WorkflowAction(
name="fail",
action_type=ActionType.FS,
status=ActionStatus.PREP
)
workflow_job.actions = {
"start": start_action,
"main-action": main_action,
"end": end_action,
"fail": fail_action
}
with self.workflow_lock:
self.workflow_jobs[job_id] = workflow_job
self.stats['total_workflows'] += 1
return {
'status': 'success',
'job_id': job_id,
'message': f'Workflow submitted successfully'
}
def start_workflow(self, job_id: str) -> Dict[str, Any]:
"""
启动工作流
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 启动结果
"""
if job_id not in self.workflow_jobs:
return {'status': 'error', 'message': f'Workflow {job_id} not found'}
workflow_job = self.workflow_jobs[job_id]
if workflow_job.status != WorkflowStatus.PREP:
return {'status': 'error', 'message': f'Workflow {job_id} is not in PREP state'}
with self.workflow_lock:
workflow_job.status = WorkflowStatus.RUNNING
workflow_job.start_time = datetime.now()
workflow_job.run += 1
self.stats['running_workflows'] += 1
# 启动第一个动作
self._execute_workflow_action(job_id, "start")
return {
'status': 'success',
'job_id': job_id,
'message': f'Workflow started successfully'
}
def _execute_workflow_action(self, job_id: str, action_name: str):
"""
执行工作流动作
Args:
job_id: 作业ID
action_name: 动作名称
"""
workflow_job = self.workflow_jobs[job_id]
action = workflow_job.actions.get(action_name)
if not action:
return
# 模拟动作执行
action.status = ActionStatus.RUNNING
action.start_time = datetime.now()
# 模拟执行时间
execution_time = random.uniform(1, 5)
time.sleep(execution_time)
# 模拟执行结果（90%成功率）
success = random.random() > 0.1
action.end_time = datetime.now()
if success:
action.status = ActionStatus.OK
# 执行下一个动作
if action.transition and action.transition != "end":
self._execute_workflow_action(job_id, action.transition)
elif action.transition == "end":
self._complete_workflow(job_id, WorkflowStatus.SUCCEEDED)
else:
action.status = ActionStatus.ERROR
action.error_code = "E0001"
action.error_message = "Action execution failed"
# 执行错误转换
if action.error_transition:
if action.error_transition == "fail":
self._complete_workflow(job_id, WorkflowStatus.FAILED)
else:
self._execute_workflow_action(job_id, action.error_transition)
def _complete_workflow(self, job_id: str, status: WorkflowStatus):
"""
完成工作流
Args:
job_id: 作业ID
status: 最终状态
"""
workflow_job = self.workflow_jobs[job_id]
with self.workflow_lock:
workflow_job.status = status
workflow_job.end_time = datetime.now()
self.stats['running_workflows'] -= 1
if status == WorkflowStatus.SUCCEEDED:
self.stats['succeeded_workflows'] += 1
elif status == WorkflowStatus.FAILED:
self.stats['failed_workflows'] += 1
def get_workflow_job(self, job_id: str) -> Dict[str, Any]:
"""
获取工作流作业信息
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业信息
"""
if job_id not in self.workflow_jobs:
return {'status': 'error', 'message': f'Workflow {job_id} not found'}
workflow_job = self.workflow_jobs[job_id]
# 获取动作信息
actions_info = []
for action_name, action in workflow_job.actions.items():
actions_info.append({
'name': action.name,
'type': action.action_type.value,
'status': action.status.value,
'start_time': action.start_time.isoformat() if action.start_time else None,
'end_time': action.end_time.isoformat() if action.end_time else None,
'transition': action.transition,
'error_transition': action.error_transition,
'retry_max': action.retry_max,
'error_code': action.error_code,
'error_message': action.error_message
})
return {
'status': 'success',
'workflow': {
'id': workflow_job.id,
'app_name': workflow_job.app_name,
'app_path': workflow_job.app_path,
'status': workflow_job.status.value,
'user': workflow_job.user,
'group': workflow_job.group,
'start_time': workflow_job.start_time.isoformat() if workflow_job.start_time else None,
'end_time': workflow_job.end_time.isoformat() if workflow_job.end_time else None,
'created_time': workflow_job.created_time.isoformat(),
'last_modified_time': workflow_job.last_modified_time.isoformat(),
'run': workflow_job.run,
'actions': actions_info,
'parent_id': workflow_job.parent_id,
'external_id': workflow_job.external_id
}
}
def kill_workflow(self, job_id: str) -> Dict[str, Any]:
"""
终止工作流
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 终止结果
"""
if job_id not in self.workflow_jobs:
return {'status': 'error', 'message': f'Workflow {job_id} not found'}
workflow_job = self.workflow_jobs[job_id]
if workflow_job.status not in [WorkflowStatus.RUNNING, WorkflowStatus.SUSPENDED]:
return {'status': 'error', 'message': f'Workflow {job_id} is not in a killable state'}
with self.workflow_lock:
workflow_job.status = WorkflowStatus.KILLED
workflow_job.end_time = datetime.now()
# 终止所有运行中的动作
for action in workflow_job.actions.values():
if action.status == ActionStatus.RUNNING:
action.status = ActionStatus.KILLED
action.end_time = datetime.now()
if workflow_job.status == WorkflowStatus.RUNNING:
self.stats['running_workflows'] -= 1
return {
'status': 'success',
'job_id': job_id,
'message': f'Workflow killed successfully'
}
def suspend_workflow(self, job_id: str) -> Dict[str, Any]:
"""
暂停工作流
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 暂停结果
"""
if job_id not in self.workflow_jobs:
return {'status': 'error', 'message': f'Workflow {job_id} not found'}
workflow_job = self.workflow_jobs[job_id]
if workflow_job.status != WorkflowStatus.RUNNING:
return {'status': 'error', 'message': f'Workflow {job_id} is not running'}
with self.workflow_lock:
workflow_job.status = WorkflowStatus.SUSPENDED
workflow_job.last_modified_time = datetime.now()
return {
'status': 'success',
'job_id': job_id,
'message': f'Workflow suspended successfully'
}
def resume_workflow(self, job_id: str) -> Dict[str, Any]:
"""
恢复工作流
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 恢复结果
"""
if job_id not in self.workflow_jobs:
return {'status': 'error', 'message': f'Workflow {job_id} not found'}
workflow_job = self.workflow_jobs[job_id]
if workflow_job.status != WorkflowStatus.SUSPENDED:
return {'status': 'error', 'message': f'Workflow {job_id} is not suspended'}
with self.workflow_lock:
workflow_job.status = WorkflowStatus.RUNNING
workflow_job.last_modified_time = datetime.now()
return {
'status': 'success',
'job_id': job_id,
'message': f'Workflow resumed successfully'
}
def submit_coordinator(self, app_path: str, config: Dict[str, Any],
user: str = "hadoop") -> Dict[str, Any]:
"""
提交协调器
Args:
app_path: 应用路径
config: 配置参数
user: 用户名
Returns:
Dict[str, Any]: 提交结果
"""
job_id = f"{len(self.coordinator_jobs)+1:07d}-{datetime.now().strftime('%Y%m%d%H%M%S')}-oozie-C"
# 解析应用名称
app_name = app_path.split('/')[-1] if app_path else "coordinator"
# 创建协调器作业
coordinator_job = CoordinatorJob(
id=job_id,
app_name=app_name,
app_path=app_path,
user=user,
group=user,
start_time=datetime.now(),
end_time=datetime.now() + timedelta(days=30),
frequency=config.get('frequency', '${coord:days(1)}'),
concurrency=config.get('concurrency', 1),
execution=config.get('execution', 'FIFO'),
timeout=config.get('timeout', 120),
conf=json.dumps(config)
)
with self.coordinator_lock:
self.coordinator_jobs[job_id] = coordinator_job
self.stats['total_coordinators'] += 1
return {
'status': 'success',
'job_id': job_id,
'message': f'Coordinator submitted successfully'
}
def start_coordinator(self, job_id: str) -> Dict[str, Any]:
"""
启动协调器
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 启动结果
"""
if job_id not in self.coordinator_jobs:
return {'status': 'error', 'message': f'Coordinator {job_id} not found'}
coordinator_job = self.coordinator_jobs[job_id]
if coordinator_job.status != CoordinatorStatus.PREP:
return {'status': 'error', 'message': f'Coordinator {job_id} is not in PREP state'}
with self.coordinator_lock:
coordinator_job.status = CoordinatorStatus.RUNNING
coordinator_job.last_modified_time = datetime.now()
coordinator_job.next_materialized_time = datetime.now() + timedelta(days=1)
self.stats['running_coordinators'] += 1
return {
'status': 'success',
'job_id': job_id,
'message': f'Coordinator started successfully'
}
def get_coordinator_job(self, job_id: str) -> Dict[str, Any]:
"""
获取协调器作业信息
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业信息
"""
if job_id not in self.coordinator_jobs:
return {'status': 'error', 'message': f'Coordinator {job_id} not found'}
coordinator_job = self.coordinator_jobs[job_id]
# 获取动作信息
actions_info = []
for action_id, action in coordinator_job.actions.items():
actions_info.append({
'id': action.id,
'job_id': action.job_id,
'status': action.status.value,
'external_id': action.external_id,
'nominal_time': action.nominal_time.isoformat() if action.nominal_time else None,
'created_time': action.created_time.isoformat(),
'missing_dependencies': action.missing_dependencies,
'timeout': action.timeout,
'error_code': action.error_code,
'error_message': action.error_message
})
return {
'status': 'success',
'coordinator': {
'id': coordinator_job.id,
'app_name': coordinator_job.app_name,
'app_path': coordinator_job.app_path,
'status': coordinator_job.status.value,
'user': coordinator_job.user,
'group': coordinator_job.group,
'start_time': coordinator_job.start_time.isoformat() if coordinator_job.start_time else None,
'end_time': coordinator_job.end_time.isoformat() if coordinator_job.end_time else None,
'pause_time': coordinator_job.pause_time.isoformat() if coordinator_job.pause_time else None,
'created_time': coordinator_job.created_time.isoformat(),
'last_modified_time': coordinator_job.last_modified_time.isoformat(),
'next_materialized_time': coordinator_job.next_materialized_time.isoformat() if coordinator_job.next_materialized_time else None,
'frequency': coordinator_job.frequency,
'time_zone': coordinator_job.time_zone,
'concurrency': coordinator_job.concurrency,
'execution': coordinator_job.execution,
'timeout': coordinator_job.timeout,
'actions': actions_info,
'mat_throttling': coordinator_job.mat_throttling
}
}
def submit_bundle(self, app_path: str, config: Dict[str, Any],
user: str = "hadoop") -> Dict[str, Any]:
"""
提交Bundle
Args:
app_path: 应用路径
config: 配置参数
user: 用户名
Returns:
Dict[str, Any]: 提交结果
"""
job_id = f"{len(self.bundle_jobs)+1:07d}-{datetime.now().strftime('%Y%m%d%H%M%S')}-oozie-B"
# 解析应用名称
app_name = app_path.split('/')[-1] if app_path else "bundle"
# 创建Bundle作业
bundle_job = BundleJob(
id=job_id,
app_name=app_name,
app_path=app_path,
user=user,
group=user,
start_time=datetime.now(),
end_time=datetime.now() + timedelta(days=90),
coordinators=config.get('coordinators', []),
conf=json.dumps(config)
)
with self.bundle_lock:
self.bundle_jobs[job_id] = bundle_job
self.stats['total_bundles'] += 1
return {
'status': 'success',
'job_id': job_id,
'message': f'Bundle submitted successfully'
}
def start_bundle(self, job_id: str) -> Dict[str, Any]:
"""
启动Bundle
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 启动结果
"""
if job_id not in self.bundle_jobs:
return {'status': 'error', 'message': f'Bundle {job_id} not found'}
bundle_job = self.bundle_jobs[job_id]
if bundle_job.status != BundleStatus.PREP:
return {'status': 'error', 'message': f'Bundle {job_id} is not in PREP state'}
with self.bundle_lock:
bundle_job.status = BundleStatus.RUNNING
bundle_job.kick_off_time = datetime.now()
self.stats['running_bundles'] += 1
return {
'status': 'success',
'job_id': job_id,
'message': f'Bundle started successfully'
}
def get_bundle_job(self, job_id: str) -> Dict[str, Any]:
"""
获取Bundle作业信息
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业信息
"""
if job_id not in self.bundle_jobs:
return {'status': 'error', 'message': f'Bundle {job_id} not found'}
bundle_job = self.bundle_jobs[job_id]
return {
'status': 'success',
'bundle': {
'id': bundle_job.id,
'app_name': bundle_job.app_name,
'app_path': bundle_job.app_path,
'status': bundle_job.status.value,
'user': bundle_job.user,
'group': bundle_job.group,
'start_time': bundle_job.start_time.isoformat() if bundle_job.start_time else None,
'end_time': bundle_job.end_time.isoformat() if bundle_job.end_time else None,
'pause_time': bundle_job.pause_time.isoformat() if bundle_job.pause_time else None,
'created_time': bundle_job.created_time.isoformat(),
'last_modified_time': bundle_job.last_modified_time.isoformat(),
'kick_off_time': bundle_job.kick_off_time.isoformat() if bundle_job.kick_off_time else None,
'time_zone': bundle_job.time_zone,
'coordinators': bundle_job.coordinators
}
}
def get_jobs(self, job_type: str = "workflow", status: Optional[str] = None,
user: Optional[str] = None, limit: int = 50) -> Dict[str, Any]:
"""
获取作业列表
Args:
job_type: 作业类型 (workflow, coordinator, bundle)
status: 状态过滤
user: 用户过滤
limit: 限制数量
Returns:
Dict[str, Any]: 作业列表
"""
jobs_info = []
count = 0
if job_type == "workflow":
for job_id, job in self.workflow_jobs.items():
if count >= limit:
break
# 应用过滤条件
if status and job.status.value != status:
continue
if user and job.user != user:
continue
jobs_info.append({
'id': job.id,
'app_name': job.app_name,
'status': job.status.value,
'user': job.user,
'group': job.group,
'start_time': job.start_time.isoformat() if job.start_time else None,
'end_time': job.end_time.isoformat() if job.end_time else None,
'created_time': job.created_time.isoformat(),
'run': job.run
})
count += 1
elif job_type == "coordinator":
for job_id, job in self.coordinator_jobs.items():
if count >= limit:
break
# 应用过滤条件
if status and job.status.value != status:
continue
if user and job.user != user:
continue
jobs_info.append({
'id': job.id,
'app_name': job.app_name,
'status': job.status.value,
'user': job.user,
'group': job.group,
'start_time': job.start_time.isoformat() if job.start_time else None,
'end_time': job.end_time.isoformat() if job.end_time else None,
'created_time': job.created_time.isoformat(),
'frequency': job.frequency,
'concurrency': job.concurrency
})
count += 1
elif job_type == "bundle":
for job_id, job in self.bundle_jobs.items():
if count >= limit:
break
# 应用过滤条件
if status and job.status.value != status:
continue
if user and job.user != user:
continue
jobs_info.append({
'id': job.id,
'app_name': job.app_name,
'status': job.status.value,
'user': job.user,
'group': job.group,
'start_time': job.start_time.isoformat() if job.start_time else None,
'end_time': job.end_time.isoformat() if job.end_time else None,
'created_time': job.created_time.isoformat(),
'coordinators_count': len(job.coordinators)
})
count += 1
return {
'status': 'success',
'jobs': jobs_info,
'total': len(jobs_info),
'job_type': job_type
}
def get_server_status(self) -> Dict[str, Any]:
"""
获取服务器状态
Returns:
Dict[str, Any]: 服务器状态
"""
return {
'server_url': self.server_url,
'is_running': self.is_running,
'build_version': self.build_version,
'system_mode': self.system_mode,
'java_version': self.java_version,
'os_name': self.os_name,
'os_version': self.os_version,
'stats': self.stats,
'config': self.config,
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Oozie服务器
oozie = OozieServer("http://localhost:11000/oozie")
print("=== Apache Oozie工作流引擎示例 ===")
# 提交工作流
print("\n=== 提交工作流 ===")
workflow_config = {
'jobTracker': 'localhost:8032',
'nameNode': 'hdfs://localhost:9000',
'queueName': 'default',
'inputDir': '/user/input',
'outputDir': '/user/output'
}
submit_result = oozie.submit_workflow(
app_path="/user/oozie/workflows/data-processing",
config=workflow_config,
user="hadoop"
)
print(f"提交结果: {submit_result}")
if submit_result['status'] == 'success':
workflow_id = submit_result['job_id']
# 启动工作流
print("\n=== 启动工作流 ===")
start_result = oozie.start_workflow(workflow_id)
print(f"启动结果: {start_result}")
# 等待一段时间
time.sleep(2)
# 获取工作流信息
print("\n=== 工作流信息 ===")
workflow_info = oozie.get_workflow_job(workflow_id)
if workflow_info['status'] == 'success':
workflow = workflow_info['workflow']
print(f"工作流ID: {workflow['id']}")
print(f"应用名称: {workflow['app_name']}")
print(f"状态: {workflow['status']}")
print(f"用户: {workflow['user']}")
print(f"开始时间: {workflow['start_time']}")
print(f"运行次数: {workflow['run']}")
print("动作列表:")
for action in workflow['actions']:
print(f" - {action['name']}: {action['status']} ({action['type']})")
if action['transition']:
print(f" 成功转换: {action['transition']}")
if action['error_transition']:
print(f" 错误转换: {action['error_transition']}")
# 提交协调器
print("\n=== 提交协调器 ===")
coordinator_config = {
'frequency': '${coord:days(1)}',
'concurrency': 1,
'execution': 'FIFO',
'timeout': 120,
'workflowPath': '/user/oozie/workflows/daily-processing'
}
coord_submit_result = oozie.submit_coordinator(
app_path="/user/oozie/coordinators/daily-processing",
config=coordinator_config,
user="hadoop"
)
print(f"协调器提交结果: {coord_submit_result}")
if coord_submit_result['status'] == 'success':
coordinator_id = coord_submit_result['job_id']
# 启动协调器
coord_start_result = oozie.start_coordinator(coordinator_id)
print(f"协调器启动结果: {coord_start_result}")
# 获取协调器信息
print("\n=== 协调器信息 ===")
coord_info = oozie.get_coordinator_job(coordinator_id)
if coord_info['status'] == 'success':
coordinator = coord_info['coordinator']
print(f"协调器ID: {coordinator['id']}")
print(f"应用名称: {coordinator['app_name']}")
print(f"状态: {coordinator['status']}")
print(f"频率: {coordinator['frequency']}")
print(f"并发数: {coordinator['concurrency']}")
print(f"执行策略: {coordinator['execution']}")
print(f"下次物化时间: {coordinator['next_materialized_time']}")
# 提交Bundle
print("\n=== 提交Bundle ===")
bundle_config = {
'coordinators': [coordinator_id] if coord_submit_result['status'] == 'success' else [],
'kickOffTime': datetime.now().isoformat()
}
bundle_submit_result = oozie.submit_bundle(
app_path="/user/oozie/bundles/data-pipeline",
config=bundle_config,
user="hadoop"
)
print(f"Bundle提交结果: {bundle_submit_result}")
if bundle_submit_result['status'] == 'success':
bundle_id = bundle_submit_result['job_id']
# 启动Bundle
bundle_start_result = oozie.start_bundle(bundle_id)
print(f"Bundle启动结果: {bundle_start_result}")
# 获取Bundle信息
print("\n=== Bundle信息 ===")
bundle_info = oozie.get_bundle_job(bundle_id)
if bundle_info['status'] == 'success':
bundle = bundle_info['bundle']
print(f"Bundle ID: {bundle['id']}")
print(f"应用名称: {bundle['app_name']}")
print(f"状态: {bundle['status']}")
print(f"协调器数量: {len(bundle['coordinators'])}")
print(f"启动时间: {bundle['kick_off_time']}")
# 获取作业列表
print("\n=== 工作流作业列表 ===")
workflows = oozie.get_jobs(job_type="workflow", limit=10)
print(f"工作流总数: {workflows['total']}")
for job in workflows['jobs']:
print(f" - {job['id']}: {job['app_name']} ({job['status']})")
print("\n=== 协调器作业列表 ===")
coordinators = oozie.get_jobs(job_type="coordinator", limit=10)
print(f"协调器总数: {coordinators['total']}")
for job in coordinators['jobs']:
print(f" - {job['id']}: {job['app_name']} ({job['status']}) - {job['frequency']}")
# 获取服务器状态
print("\n=== 服务器状态 ===")
status = oozie.get_server_status()
print(f"服务器URL: {status['server_url']}")
print(f"运行状态: {status['is_running']}")
print(f"版本: {status['build_version']}")
print(f"系统模式: {status['system_mode']}")
print(f"Java版本: {status['java_version']}")
print("统计信息:")
for key, value in status['stats'].items():
print(f" {key}: {value}")
8.2 Apache Flume详解
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import re
class FlumeComponentType(Enum):
"""Flume组件类型"""
SOURCE = "source"
SINK = "sink"
CHANNEL = "channel"
INTERCEPTOR = "interceptor"
SERIALIZER = "serializer"
SINK_PROCESSOR = "sink_processor"
class FlumeComponentStatus(Enum):
"""Flume组件状态"""
IDLE = "IDLE" # 空闲
START = "START" # 启动中
STOP = "STOP" # 停止中
ERROR = "ERROR" # 错误
class SourceType(Enum):
"""Source类型"""
SPOOLDIR = "spooldir" # 目录监控
TAILDIR = "taildir" # 文件尾部监控
KAFKA = "kafka" # Kafka消费者
HTTP = "http" # HTTP接收器
NETCAT = "netcat" # Netcat
EXEC = "exec" # 命令执行
JMS = "jms" # JMS消息
AVRO = "avro" # Avro RPC
THRIFT = "thrift" # Thrift RPC
SEQUENCE_GENERATOR = "seq" # 序列生成器
class SinkType(Enum):
"""Sink类型"""
HDFS = "hdfs" # HDFS写入
HBASE = "hbase" # HBase写入
KAFKA = "kafka" # Kafka生产者
ELASTICSEARCH = "elasticsearch" # Elasticsearch
SOLR = "solr" # Solr搜索
LOGGER = "logger" # 日志输出
NULL = "null" # 空输出
AVRO = "avro" # Avro RPC
THRIFT = "thrift" # Thrift RPC
FILE_ROLL = "file_roll" # 文件滚动
HIVE = "hive" # Hive写入
class ChannelType(Enum):
"""Channel类型"""
MEMORY = "memory" # 内存通道
FILE = "file" # 文件通道
SPILLABLE_MEMORY = "spillablememory" # 可溢出内存通道
JDBC = "jdbc" # JDBC通道
KAFKA = "kafka" # Kafka通道
PSEUDO_TRANSACTION = "pseudotxn" # 伪事务通道
class EventStatus(Enum):
"""事件状态"""
READY = "READY" # 就绪
INFLIGHT = "INFLIGHT" # 传输中
TAKE = "TAKE" # 已取出
COMMIT = "COMMIT" # 已提交
ROLLBACK = "ROLLBACK" # 已回滚
@dataclass
class FlumeEvent:
"""Flume事件"""
event_id: str
headers: Dict[str, str] = field(default_factory=dict)
body: bytes = b""
timestamp: datetime = field(default_factory=datetime.now)
status: EventStatus = EventStatus.READY
source_name: str = ""
channel_name: str = ""
sink_name: str = ""
retry_count: int = 0
max_retries: int = 3
def get_header(self, key: str, default: str = "") -> str:
"""获取头部信息"""
return self.headers.get(key, default)
def set_header(self, key: str, value: str):
"""设置头部信息"""
self.headers[key] = value
def get_body_as_string(self, encoding: str = 'utf-8') -> str:
"""获取事件体字符串"""
try:
return self.body.decode(encoding)
except UnicodeDecodeError:
return self.body.decode('utf-8', errors='ignore')
@dataclass
class FlumeTransaction:
"""Flume事务"""
transaction_id: str
channel_name: str
events: List[FlumeEvent] = field(default_factory=list)
start_time: datetime = field(default_factory=datetime.now)
end_time: Optional[datetime] = None
is_committed: bool = False
is_rolled_back: bool = False
def add_event(self, event: FlumeEvent):
"""添加事件到事务"""
self.events.append(event)
def commit(self):
"""提交事务"""
self.is_committed = True
self.end_time = datetime.now()
for event in self.events:
event.status = EventStatus.COMMIT
def rollback(self):
"""回滚事务"""
self.is_rolled_back = True
self.end_time = datetime.now()
for event in self.events:
event.status = EventStatus.ROLLBACK
@dataclass
class FlumeSource:
"""Flume Source"""
name: str
source_type: SourceType
channels: List[str] = field(default_factory=list)
config: Dict[str, Any] = field(default_factory=dict)
status: FlumeComponentStatus = FlumeComponentStatus.IDLE
events_received: int = 0
events_accepted: int = 0
events_rejected: int = 0
last_activity: Optional[datetime] = None
error_count: int = 0
error_message: str = ""
def add_channel(self, channel_name: str):
"""添加通道"""
if channel_name not in self.channels:
self.channels.append(channel_name)
def remove_channel(self, channel_name: str):
"""移除通道"""
if channel_name in self.channels:
self.channels.remove(channel_name)
@dataclass
class FlumeSink:
"""Flume Sink"""
name: str
sink_type: SinkType
channel: str
config: Dict[str, Any] = field(default_factory=dict)
status: FlumeComponentStatus = FlumeComponentStatus.IDLE
events_drained: int = 0
events_failed: int = 0
connection_failures: int = 0
batch_size: int = 100
batch_timeout: int = 3000 # 毫秒
last_activity: Optional[datetime] = None
error_count: int = 0
error_message: str = ""
@dataclass
class FlumeChannel:
"""Flume Channel"""
name: str
channel_type: ChannelType
config: Dict[str, Any] = field(default_factory=dict)
status: FlumeComponentStatus = FlumeComponentStatus.IDLE
capacity: int = 10000
transaction_capacity: int = 1000
events: deque = field(default_factory=deque)
transactions: Dict[str, FlumeTransaction] = field(default_factory=dict)
events_put: int = 0
events_take: int = 0
channel_size: int = 0
channel_fill_percentage: float = 0.0
last_activity: Optional[datetime] = None
def put_event(self, event: FlumeEvent, transaction_id: str) -> bool:
"""放入事件"""
if len(self.events) >= self.capacity:
return False
if transaction_id not in self.transactions:
self.transactions[transaction_id] = FlumeTransaction(
transaction_id=transaction_id,
channel_name=self.name
)
transaction = self.transactions[transaction_id]
if len(transaction.events) >= self.transaction_capacity:
return False
event.status = EventStatus.INFLIGHT
event.channel_name = self.name
transaction.add_event(event)
self.events_put += 1
self.last_activity = datetime.now()
return True
def take_event(self, transaction_id: str) -> Optional[FlumeEvent]:
"""取出事件"""
if not self.events:
return None
if transaction_id not in self.transactions:
self.transactions[transaction_id] = FlumeTransaction(
transaction_id=transaction_id,
channel_name=self.name
)
event = self.events.popleft()
event.status = EventStatus.TAKE
transaction = self.transactions[transaction_id]
transaction.add_event(event)
self.events_take += 1
self.channel_size = len(self.events)
self.channel_fill_percentage = (self.channel_size / self.capacity) * 100
self.last_activity = datetime.now()
return event
def commit_transaction(self, transaction_id: str) -> bool:
"""提交事务"""
if transaction_id not in self.transactions:
return False
transaction = self.transactions[transaction_id]
transaction.commit()
# 对于put操作，将事件添加到通道
for event in transaction.events:
if event.status == EventStatus.INFLIGHT:
self.events.append(event)
self.channel_size = len(self.events)
self.channel_fill_percentage = (self.channel_size / self.capacity) * 100
del self.transactions[transaction_id]
return True
def rollback_transaction(self, transaction_id: str) -> bool:
"""回滚事务"""
if transaction_id not in self.transactions:
return False
transaction = self.transactions[transaction_id]
transaction.rollback()
# 对于take操作，将事件放回通道
for event in transaction.events:
if event.status == EventStatus.TAKE:
self.events.appendleft(event)
self.channel_size = len(self.events)
self.channel_fill_percentage = (self.channel_size / self.capacity) * 100
del self.transactions[transaction_id]
return True
@dataclass
class FlumeAgent:
"""Flume Agent"""
name: str
host: str = "localhost"
port: int = 41414
sources: Dict[str, FlumeSource] = field(default_factory=dict)
sinks: Dict[str, FlumeSink] = field(default_factory=dict)
channels: Dict[str, FlumeChannel] = field(default_factory=dict)
is_running: bool = False
start_time: Optional[datetime] = None
config_file: str = ""
jvm_metrics: Dict[str, Any] = field(default_factory=dict)
class FlumeCluster:
"""
Apache Flume数据收集集群
"""
def __init__(self, cluster_name: str = "flume-cluster"):
self.cluster_name = cluster_name
# 数据存储
self.agents = {} # agent_name -> FlumeAgent
# 集群状态
self.is_running = True
self.version = "1.11.0"
self.java_version = "1.8.0_281"
# 配置
self.config = {
'flume.monitoring.type': 'http',
'flume.monitoring.port': 34545,
'flume.root.logger': 'INFO,console',
'flume.log.dir': './logs',
'flume.log.file': 'flume.log'
}
# 线程锁
self.agents_lock = threading.Lock()
# 统计信息
self.stats = {
'total_agents': 0,
'running_agents': 0,
'total_sources': 0,
'total_sinks': 0,
'total_channels': 0,
'total_events_processed': 0,
'total_events_failed': 0,
'average_throughput': 0.0
}
# 初始化示例代理
self._create_example_agents()
def _create_example_agents(self):
"""创建示例代理"""
# 创建示例代理
agent1 = FlumeAgent(
name="web-log-agent",
host="node1.example.com",
port=41414
)
agent2 = FlumeAgent(
name="app-log-agent",
host="node2.example.com",
port=41414
)
with self.agents_lock:
self.agents[agent1.name] = agent1
self.agents[agent2.name] = agent2
self._update_stats()
def _update_stats(self):
"""更新统计信息"""
self.stats['total_agents'] = len(self.agents)
self.stats['running_agents'] = len([a for a in self.agents.values() if a.is_running])
self.stats['total_sources'] = sum(len(a.sources) for a in self.agents.values())
self.stats['total_sinks'] = sum(len(a.sinks) for a in self.agents.values())
self.stats['total_channels'] = sum(len(a.channels) for a in self.agents.values())
def create_agent(self, agent_name: str, host: str = "localhost", port: int = 41414) -> Dict[str, Any]:
"""
创建Flume代理
Args:
agent_name: 代理名称
host: 主机地址
port: 端口号
Returns:
Dict[str, Any]: 创建结果
"""
if agent_name in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} already exists'}
agent = FlumeAgent(
name=agent_name,
host=host,
port=port
)
with self.agents_lock:
self.agents[agent_name] = agent
self._update_stats()
return {
'status': 'success',
'agent_name': agent_name,
'host': host,
'port': port,
'message': f'Agent {agent_name} created successfully'
}
def start_agent(self, agent_name: str) -> Dict[str, Any]:
"""
启动代理
Args:
agent_name: 代理名称
Returns:
Dict[str, Any]: 启动结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if agent.is_running:
return {'status': 'error', 'message': f'Agent {agent_name} is already running'}
agent.is_running = True
agent.start_time = datetime.now()
# 启动所有组件
for source in agent.sources.values():
source.status = FlumeComponentStatus.START
for sink in agent.sinks.values():
sink.status = FlumeComponentStatus.START
for channel in agent.channels.values():
channel.status = FlumeComponentStatus.START
with self.agents_lock:
self._update_stats()
return {
'status': 'success',
'agent_name': agent_name,
'start_time': agent.start_time.isoformat(),
'message': f'Agent {agent_name} started successfully'
}
def stop_agent(self, agent_name: str) -> Dict[str, Any]:
"""
停止代理
Args:
agent_name: 代理名称
Returns:
Dict[str, Any]: 停止结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if not agent.is_running:
return {'status': 'error', 'message': f'Agent {agent_name} is not running'}
agent.is_running = False
# 停止所有组件
for source in agent.sources.values():
source.status = FlumeComponentStatus.STOP
for sink in agent.sinks.values():
sink.status = FlumeComponentStatus.STOP
for channel in agent.channels.values():
channel.status = FlumeComponentStatus.STOP
with self.agents_lock:
self._update_stats()
return {
'status': 'success',
'agent_name': agent_name,
'message': f'Agent {agent_name} stopped successfully'
}
def add_source(self, agent_name: str, source_name: str, source_type: SourceType,
channels: List[str], config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
添加Source
Args:
agent_name: 代理名称
source_name: Source名称
source_type: Source类型
channels: 通道列表
config: 配置
Returns:
Dict[str, Any]: 添加结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if source_name in agent.sources:
return {'status': 'error', 'message': f'Source {source_name} already exists'}
source = FlumeSource(
name=source_name,
source_type=source_type,
channels=channels.copy(),
config=config or {}
)
agent.sources[source_name] = source
with self.agents_lock:
self._update_stats()
return {
'status': 'success',
'agent_name': agent_name,
'source_name': source_name,
'source_type': source_type.value,
'channels': channels,
'message': f'Source {source_name} added successfully'
}
def add_sink(self, agent_name: str, sink_name: str, sink_type: SinkType,
channel: str, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
添加Sink
Args:
agent_name: 代理名称
sink_name: Sink名称
sink_type: Sink类型
channel: 通道名称
config: 配置
Returns:
Dict[str, Any]: 添加结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if sink_name in agent.sinks:
return {'status': 'error', 'message': f'Sink {sink_name} already exists'}
sink = FlumeSink(
name=sink_name,
sink_type=sink_type,
channel=channel,
config=config or {}
)
agent.sinks[sink_name] = sink
with self.agents_lock:
self._update_stats()
return {
'status': 'success',
'agent_name': agent_name,
'sink_name': sink_name,
'sink_type': sink_type.value,
'channel': channel,
'message': f'Sink {sink_name} added successfully'
}
def add_channel(self, agent_name: str, channel_name: str, channel_type: ChannelType,
config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""
添加Channel
Args:
agent_name: 代理名称
channel_name: Channel名称
channel_type: Channel类型
config: 配置
Returns:
Dict[str, Any]: 添加结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if channel_name in agent.channels:
return {'status': 'error', 'message': f'Channel {channel_name} already exists'}
channel_config = config or {}
capacity = channel_config.get('capacity', 10000)
transaction_capacity = channel_config.get('transactionCapacity', 1000)
channel = FlumeChannel(
name=channel_name,
channel_type=channel_type,
config=channel_config,
capacity=capacity,
transaction_capacity=transaction_capacity
)
agent.channels[channel_name] = channel
with self.agents_lock:
self._update_stats()
return {
'status': 'success',
'agent_name': agent_name,
'channel_name': channel_name,
'channel_type': channel_type.value,
'capacity': capacity,
'message': f'Channel {channel_name} added successfully'
}
def send_event(self, agent_name: str, source_name: str, headers: Dict[str, str],
body: Union[str, bytes]) -> Dict[str, Any]:
"""
发送事件
Args:
agent_name: 代理名称
source_name: Source名称
headers: 事件头部
body: 事件体
Returns:
Dict[str, Any]: 发送结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if source_name not in agent.sources:
return {'status': 'error', 'message': f'Source {source_name} not found'}
source = agent.sources[source_name]
if source.status != FlumeComponentStatus.START:
return {'status': 'error', 'message': f'Source {source_name} is not running'}
# 创建事件
event_id = f"event-{uuid.uuid4().hex[:8]}"
event_body = body.encode('utf-8') if isinstance(body, str) else body
event = FlumeEvent(
event_id=event_id,
headers=headers.copy(),
body=event_body,
source_name=source_name
)
# 发送到所有配置的通道
success_channels = []
failed_channels = []
for channel_name in source.channels:
if channel_name not in agent.channels:
failed_channels.append(channel_name)
continue
channel = agent.channels[channel_name]
transaction_id = f"txn-{uuid.uuid4().hex[:8]}"
if channel.put_event(event, transaction_id):
if channel.commit_transaction(transaction_id):
success_channels.append(channel_name)
else:
channel.rollback_transaction(transaction_id)
failed_channels.append(channel_name)
else:
failed_channels.append(channel_name)
# 更新Source统计
source.events_received += 1
source.last_activity = datetime.now()
if success_channels:
source.events_accepted += 1
else:
source.events_rejected += 1
return {
'status': 'success' if success_channels else 'error',
'event_id': event_id,
'success_channels': success_channels,
'failed_channels': failed_channels,
'message': f'Event sent to {len(success_channels)} channels'
}
def process_events(self, agent_name: str, sink_name: str, batch_size: int = 100) -> Dict[str, Any]:
"""
处理事件（Sink从Channel取出事件）
Args:
agent_name: 代理名称
sink_name: Sink名称
batch_size: 批次大小
Returns:
Dict[str, Any]: 处理结果
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
if sink_name not in agent.sinks:
return {'status': 'error', 'message': f'Sink {sink_name} not found'}
sink = agent.sinks[sink_name]
if sink.status != FlumeComponentStatus.START:
return {'status': 'error', 'message': f'Sink {sink_name} is not running'}
if sink.channel not in agent.channels:
return {'status': 'error', 'message': f'Channel {sink.channel} not found'}
channel = agent.channels[sink.channel]
transaction_id = f"txn-{uuid.uuid4().hex[:8]}"
# 取出事件批次
events = []
for _ in range(min(batch_size, sink.batch_size)):
event = channel.take_event(transaction_id)
if event is None:
break
events.append(event)
if not events:
return {
'status': 'success',
'events_processed': 0,
'message': 'No events to process'
}
# 模拟事件处理（90%成功率）
success = random.random() > 0.1
if success:
# 提交事务
channel.commit_transaction(transaction_id)
sink.events_drained += len(events)
sink.last_activity = datetime.now()
return {
'status': 'success',
'events_processed': len(events),
'sink_type': sink.sink_type.value,
'message': f'Successfully processed {len(events)} events'
}
else:
# 回滚事务
channel.rollback_transaction(transaction_id)
sink.events_failed += len(events)
sink.error_count += 1
sink.error_message = "Failed to write events to destination"
return {
'status': 'error',
'events_processed': 0,
'events_failed': len(events),
'message': 'Failed to process events'
}
def get_agent_status(self, agent_name: str) -> Dict[str, Any]:
"""
获取代理状态
Args:
agent_name: 代理名称
Returns:
Dict[str, Any]: 代理状态
"""
if agent_name not in self.agents:
return {'status': 'error', 'message': f'Agent {agent_name} not found'}
agent = self.agents[agent_name]
# 收集Source信息
sources_info = []
for source_name, source in agent.sources.items():
sources_info.append({
'name': source.name,
'type': source.source_type.value,
'status': source.status.value,
'channels': source.channels,
'events_received': source.events_received,
'events_accepted': source.events_accepted,
'events_rejected': source.events_rejected,
'last_activity': source.last_activity.isoformat() if source.last_activity else None
})
# 收集Sink信息
sinks_info = []
for sink_name, sink in agent.sinks.items():
sinks_info.append({
'name': sink.name,
'type': sink.sink_type.value,
'status': sink.status.value,
'channel': sink.channel,
'events_drained': sink.events_drained,
'events_failed': sink.events_failed,
'connection_failures': sink.connection_failures,
'last_activity': sink.last_activity.isoformat() if sink.last_activity else None
})
# 收集Channel信息
channels_info = []
for channel_name, channel in agent.channels.items():
channels_info.append({
'name': channel.name,
'type': channel.channel_type.value,
'status': channel.status.value,
'capacity': channel.capacity,
'channel_size': channel.channel_size,
'channel_fill_percentage': channel.channel_fill_percentage,
'events_put': channel.events_put,
'events_take': channel.events_take,
'active_transactions': len(channel.transactions),
'last_activity': channel.last_activity.isoformat() if channel.last_activity else None
})
return {
'status': 'success',
'agent': {
'name': agent.name,
'host': agent.host,
'port': agent.port,
'is_running': agent.is_running,
'start_time': agent.start_time.isoformat() if agent.start_time else None,
'sources': sources_info,
'sinks': sinks_info,
'channels': channels_info
}
}
def list_agents(self) -> Dict[str, Any]:
"""
列出所有代理
Returns:
Dict[str, Any]: 代理列表
"""
agents_info = []
for agent_name, agent in self.agents.items():
agents_info.append({
'name': agent.name,
'host': agent.host,
'port': agent.port,
'is_running': agent.is_running,
'start_time': agent.start_time.isoformat() if agent.start_time else None,
'sources_count': len(agent.sources),
'sinks_count': len(agent.sinks),
'channels_count': len(agent.channels)
})
return {
'status': 'success',
'agents': agents_info,
'total': len(agents_info)
}
def get_cluster_status(self) -> Dict[str, Any]:
"""
获取集群状态
Returns:
Dict[str, Any]: 集群状态
"""
# 计算总体统计
total_events_processed = 0
total_events_failed = 0
for agent in self.agents.values():
for source in agent.sources.values():
total_events_processed += source.events_accepted
for sink in agent.sinks.values():
total_events_processed += sink.events_drained
total_events_failed += sink.events_failed
self.stats['total_events_processed'] = total_events_processed
self.stats['total_events_failed'] = total_events_failed
return {
'cluster_name': self.cluster_name,
'is_running': self.is_running,
'version': self.version,
'java_version': self.java_version,
'stats': self.stats,
'config': self.config,
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Flume集群
flume = FlumeCluster("production-flume")
print("=== Apache Flume数据收集示例 ===")
# 创建代理
print("\n=== 创建Flume代理 ===")
agent_result = flume.create_agent(
agent_name="web-server-agent",
host="web-server-01",
port=41414
)
print(f"代理创建结果: {agent_result}")
if agent_result['status'] == 'success':
agent_name = agent_result['agent_name']
# 添加Channel
print("\n=== 添加Channel ===")
channel_result = flume.add_channel(
agent_name=agent_name,
channel_name="memory-channel",
channel_type=ChannelType.MEMORY,
config={
'capacity': 10000,
'transactionCapacity': 1000
}
)
print(f"Channel添加结果: {channel_result}")
# 添加Source
print("\n=== 添加Source ===")
source_result = flume.add_source(
agent_name=agent_name,
source_name="spooldir-source",
source_type=SourceType.SPOOLDIR,
channels=["memory-channel"],
config={
'spoolDir': '/var/log/web/spool',
'channels': 'memory-channel'
}
)
print(f"Source添加结果: {source_result}")
# 添加Sink
print("\n=== 添加Sink ===")
sink_result = flume.add_sink(
agent_name=agent_name,
sink_name="hdfs-sink",
sink_type=SinkType.HDFS,
channel="memory-channel",
config={
'hdfs.path': '/flume/events/%Y/%m/%d',
'hdfs.fileType': 'DataStream',
'hdfs.writeFormat': 'Text',
'hdfs.batchSize': 1000,
'hdfs.rollInterval': 600,
'hdfs.rollSize': 268435456
}
)
print(f"Sink添加结果: {sink_result}")
# 启动代理
print("\n=== 启动代理 ===")
start_result = flume.start_agent(agent_name)
print(f"代理启动结果: {start_result}")
if start_result['status'] == 'success':
# 发送事件
print("\n=== 发送事件 ===")
for i in range(10):
event_result = flume.send_event(
agent_name=agent_name,
source_name="spooldir-source",
headers={
'timestamp': str(int(time.time())),
'host': 'web-server-01',
'source': 'access.log'
},
body=f"192.168.1.{100+i} - - [25/Dec/2023:10:00:{i:02d} +0000] \"GET /api/users HTTP/1.1\" 200 1234"
)
if i < 3: # 只打印前3个结果
print(f"事件 {i+1} 发送结果: {event_result}")
# 处理事件
print("\n=== 处理事件 ===")
for i in range(3):
process_result = flume.process_events(
agent_name=agent_name,
sink_name="hdfs-sink",
batch_size=5
)
print(f"批次 {i+1} 处理结果: {process_result}")
time.sleep(1)
# 获取代理状态
print("\n=== 代理状态 ===")
agent_status = flume.get_agent_status(agent_name)
if agent_status['status'] == 'success':
agent_info = agent_status['agent']
print(f"代理名称: {agent_info['name']}")
print(f"主机: {agent_info['host']}:{agent_info['port']}")
print(f"运行状态: {agent_info['is_running']}")
print(f"启动时间: {agent_info['start_time']}")
print("\nSources:")
for source in agent_info['sources']:
print(f" - {source['name']} ({source['type']}): {source['events_received']} received, {source['events_accepted']} accepted")
print("\nChannels:")
for channel in agent_info['channels']:
print(f" - {channel['name']} ({channel['type']}): {channel['channel_size']}/{channel['capacity']} ({channel['channel_fill_percentage']:.1f}%)")
print("\nSinks:")
for sink in agent_info['sinks']:
print(f" - {sink['name']} ({sink['type']}): {sink['events_drained']} drained, {sink['events_failed']} failed")
# 列出所有代理
print("\n=== 代理列表 ===")
agents_list = flume.list_agents()
if agents_list['status'] == 'success':
print(f"总代理数: {agents_list['total']}")
for agent in agents_list['agents']:
print(f" - {agent['name']} @ {agent['host']}:{agent['port']} - 运行: {agent['is_running']}")
# 获取集群状态
print("\n=== 集群状态 ===")
cluster_status = flume.get_cluster_status()
print(f"集群名称: {cluster_status['cluster_name']}")
print(f"版本: {cluster_status['version']}")
print(f"运行状态: {cluster_status['is_running']}")
print("统计信息:")
for key, value in cluster_status['stats'].items():
print(f" {key}: {value}")
8.3 Apache NiFi详解
from typing import Dict, List, Any, Optional, Tuple, Union, Set
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import re
from urllib.parse import urlparse
class ProcessorType(Enum):
"""处理器类型"""
GET_FILE = "GetFile" # 获取文件
PUT_FILE = "PutFile" # 写入文件
GET_HTTP = "GetHTTP" # HTTP获取
POST_HTTP = "PostHTTP" # HTTP发送
GET_KAFKA = "GetKafka" # Kafka消费
PUT_KAFKA = "PutKafka" # Kafka生产
GET_HDFS = "GetHDFS" # HDFS读取
PUT_HDFS = "PutHDFS" # HDFS写入
EXTRACT_TEXT = "ExtractText" # 文本提取
REPLACE_TEXT = "ReplaceText" # 文本替换
ROUTE_ON_ATTRIBUTE = "RouteOnAttribute" # 属性路由
UPDATE_ATTRIBUTE = "UpdateAttribute" # 更新属性
SPLIT_TEXT = "SplitText" # 文本分割
MERGE_CONTENT = "MergeContent" # 内容合并
COMPRESS_CONTENT = "CompressContent" # 内容压缩
DECOMPRESS_CONTENT = "DecompressContent" # 内容解压
CONVERT_RECORD = "ConvertRecord" # 记录转换
QUERY_RECORD = "QueryRecord" # 记录查询
EXECUTE_SQL = "ExecuteSQL" # SQL执行
PUT_SQL = "PutSQL" # SQL写入
GENERATE_FLOWFILE = "GenerateFlowFile" # 生成FlowFile
LOG_ATTRIBUTE = "LogAttribute" # 日志属性
class ProcessorState(Enum):
"""处理器状态"""
STOPPED = "STOPPED" # 停止
RUNNING = "RUNNING" # 运行
DISABLED = "DISABLED" # 禁用
INVALID = "INVALID" # 无效
VALIDATING = "VALIDATING" # 验证中
class FlowFileState(Enum):
"""FlowFile状态"""
ACTIVE = "ACTIVE" # 活跃
QUEUED = "QUEUED" # 排队
PROCESSING = "PROCESSING" # 处理中
COMPLETED = "COMPLETED" # 完成
FAILED = "FAILED" # 失败
PENALIZED = "PENALIZED" # 惩罚
EXPIRED = "EXPIRED" # 过期
class RelationshipType(Enum):
"""关系类型"""
SUCCESS = "success" # 成功
FAILURE = "failure" # 失败
ORIGINAL = "original" # 原始
MATCHED = "matched" # 匹配
UNMATCHED = "unmatched" # 不匹配
RETRY = "retry" # 重试
INVALID = "invalid" # 无效
SPLIT = "split" # 分割
MERGED = "merged" # 合并
ROUTED = "routed" # 路由
class ConnectionType(Enum):
"""连接类型"""
STANDARD = "standard" # 标准连接
LOAD_BALANCE = "load_balance" # 负载均衡
FUNNEL = "funnel" # 漏斗
SELF_LOOP = "self_loop" # 自循环
class QueuePriority(Enum):
"""队列优先级"""
FIFO = "FirstInFirstOut" # 先进先出
LIFO = "LastInFirstOut" # 后进先出
OLDEST_FIRST = "OldestFlowFileFirst" # 最旧优先
NEWEST_FIRST = "NewestFlowFileFirst" # 最新优先
LARGEST_FIRST = "LargestFlowFileFirst" # 最大优先
SMALLEST_FIRST = "SmallestFlowFileFirst" # 最小优先
@dataclass
class FlowFile:
"""NiFi FlowFile"""
uuid: str
attributes: Dict[str, str] = field(default_factory=dict)
content: bytes = b""
size: int = 0
entry_date: datetime = field(default_factory=datetime.now)
lineage_start_date: datetime = field(default_factory=datetime.now)
last_queued_date: datetime = field(default_factory=datetime.now)
queue_date_index: int = 0
state: FlowFileState = FlowFileState.ACTIVE
penalty_expiration: Optional[datetime] = None
def __post_init__(self):
if self.size == 0 and self.content:
self.size = len(self.content)
def get_attribute(self, key: str, default: str = "") -> str:
"""获取属性"""
return self.attributes.get(key, default)
def put_attribute(self, key: str, value: str):
"""设置属性"""
self.attributes[key] = value
def remove_attribute(self, key: str) -> bool:
"""移除属性"""
if key in self.attributes:
del self.attributes[key]
return True
return False
def clone(self) -> 'FlowFile':
"""克隆FlowFile"""
return FlowFile(
uuid=f"clone-{uuid.uuid4().hex[:8]}",
attributes=self.attributes.copy(),
content=self.content,
size=self.size,
entry_date=self.entry_date,
lineage_start_date=self.lineage_start_date
)
@dataclass
class Relationship:
"""处理器关系"""
name: str
description: str = ""
auto_terminate: bool = False
@dataclass
class Connection:
"""连接"""
id: str
name: str
source_id: str
destination_id: str
relationships: List[str] = field(default_factory=list)
connection_type: ConnectionType = ConnectionType.STANDARD
queue_priority: QueuePriority = QueuePriority.FIFO
flow_file_expiration: timedelta = field(default_factory=lambda: timedelta(seconds=0))
back_pressure_object_threshold: int = 10000
back_pressure_data_size_threshold: str = "1 GB"
load_balance_strategy: str = "DO_NOT_LOAD_BALANCE"
load_balance_partition_attribute: str = ""
queue: deque = field(default_factory=deque)
queued_count: int = 0
queued_size: int = 0
def enqueue(self, flow_file: FlowFile):
"""入队"""
flow_file.state = FlowFileState.QUEUED
flow_file.last_queued_date = datetime.now()
self.queue.append(flow_file)
self.queued_count += 1
self.queued_size += flow_file.size
def dequeue(self) -> Optional[FlowFile]:
"""出队"""
if not self.queue:
return None
flow_file = self.queue.popleft()
flow_file.state = FlowFileState.PROCESSING
self.queued_count -= 1
self.queued_size -= flow_file.size
return flow_file
def is_back_pressure_enabled(self) -> bool:
"""检查是否启用背压"""
return (self.queued_count >= self.back_pressure_object_threshold or
self.queued_size >= self._parse_data_size(self.back_pressure_data_size_threshold))
def _parse_data_size(self, size_str: str) -> int:
"""解析数据大小字符串"""
size_str = size_str.upper().strip()
if size_str.endswith('KB'):
return int(float(size_str[:-2]) * 1024)
elif size_str.endswith('MB'):
return int(float(size_str[:-2]) * 1024 * 1024)
elif size_str.endswith('GB'):
return int(float(size_str[:-2]) * 1024 * 1024 * 1024)
elif size_str.endswith('TB'):
return int(float(size_str[:-2]) * 1024 * 1024 * 1024 * 1024)
else:
return int(float(size_str))
@dataclass
class Processor:
"""NiFi处理器"""
id: str
name: str
processor_type: ProcessorType
state: ProcessorState = ProcessorState.STOPPED
properties: Dict[str, str] = field(default_factory=dict)
relationships: Dict[str, Relationship] = field(default_factory=dict)
incoming_connections: List[str] = field(default_factory=list)
outgoing_connections: List[str] = field(default_factory=list)
run_duration_millis: int = 0
scheduling_period: str = "0 sec"
scheduling_strategy: str = "TIMER_DRIVEN"
execution_node: str = "ALL"
penalty_duration: str = "30 sec"
yield_duration: str = "1 sec"
bulletin_level: str = "WARN"
run_schedule: Optional[str] = None
concurrent_tasks: int = 1
# 统计信息
bytes_read: int = 0
bytes_written: int = 0
bytes_transferred: int = 0
flow_files_received: int = 0
flow_files_sent: int = 0
flow_files_removed: int = 0
invocations: int = 0
processing_nanos: int = 0
def add_relationship(self, name: str, description: str = "", auto_terminate: bool = False):
"""添加关系"""
self.relationships[name] = Relationship(
name=name,
description=description,
auto_terminate=auto_terminate
)
def get_property(self, key: str, default: str = "") -> str:
"""获取属性"""
return self.properties.get(key, default)
def set_property(self, key: str, value: str):
"""设置属性"""
self.properties[key] = value
@dataclass
class ProcessGroup:
"""处理组"""
id: str
name: str
parent_group_id: Optional[str] = None
processors: Dict[str, Processor] = field(default_factory=dict)
connections: Dict[str, Connection] = field(default_factory=dict)
process_groups: Dict[str, 'ProcessGroup'] = field(default_factory=dict)
input_ports: Dict[str, Any] = field(default_factory=dict)
output_ports: Dict[str, Any] = field(default_factory=dict)
labels: Dict[str, Any] = field(default_factory=dict)
variables: Dict[str, str] = field(default_factory=dict)
parameter_context_id: Optional[str] = None
flow_file_concurrency: str = "UNBOUNDED"
flow_file_outbound_policy: str = "STREAM_WHEN_AVAILABLE"
default_flow_file_expiration: str = "0 sec"
default_back_pressure_object_threshold: int = 10000
default_back_pressure_data_size_threshold: str = "1 GB"
def add_processor(self, processor: Processor):
"""添加处理器"""
self.processors[processor.id] = processor
def add_connection(self, connection: Connection):
"""添加连接"""
self.connections[connection.id] = connection
def get_variable(self, name: str, default: str = "") -> str:
"""获取变量"""
return self.variables.get(name, default)
def set_variable(self, name: str, value: str):
"""设置变量"""
self.variables[name] = value
@dataclass
class FlowRegistry:
"""流注册表"""
id: str
name: str
url: str
description: str = ""
@dataclass
class VersionedFlow:
"""版本化流"""
registry_id: str
bucket_id: str
flow_id: str
version: int
flow_name: str
description: str = ""
comments: str = ""
class NiFiCluster:
"""
Apache NiFi数据流处理集群
"""
def __init__(self, cluster_name: str = "nifi-cluster"):
self.cluster_name = cluster_name
# 数据存储
self.process_groups = {} # group_id -> ProcessGroup
self.processors = {} # processor_id -> Processor
self.connections = {} # connection_id -> Connection
self.flow_registries = {} # registry_id -> FlowRegistry
self.versioned_flows = {} # flow_id -> VersionedFlow
# 集群状态
self.is_running = True
self.version = "1.18.0"
self.java_version = "11.0.16"
self.cluster_coordinator = "node1.example.com:11443"
# 配置
self.config = {
'nifi.web.http.port': 8080,
'nifi.web.https.port': 8443,
'nifi.cluster.is.node': True,
'nifi.cluster.node.address': 'localhost',
'nifi.cluster.node.protocol.port': 11443,
'nifi.zookeeper.connect.string': 'localhost:2181',
'nifi.state.management.embedded.zookeeper.start': True,
'nifi.flowfile.repository.implementation': 'org.apache.nifi.controller.repository.WriteAheadFlowFileRepository',
'nifi.content.repository.implementation': 'org.apache.nifi.controller.repository.FileSystemRepository',
'nifi.provenance.repository.implementation': 'org.apache.nifi.provenance.WriteAheadProvenanceRepository'
}
# 线程锁
self.cluster_lock = threading.Lock()
# 统计信息
self.stats = {
'total_processors': 0,
'running_processors': 0,
'stopped_processors': 0,
'invalid_processors': 0,
'total_connections': 0,
'total_process_groups': 0,
'active_threads': 0,
'queued_flow_files': 0,
'queued_content_size': 0,
'bytes_read_5_min': 0,
'bytes_written_5_min': 0,
'bytes_transferred_5_min': 0,
'flow_files_received_5_min': 0,
'flow_files_sent_5_min': 0
}
# 初始化根处理组
self._create_root_process_group()
# 创建示例流
self._create_example_flow()
def _create_root_process_group(self):
"""创建根处理组"""
root_group = ProcessGroup(
id="root",
name="NiFi Flow"
)
with self.cluster_lock:
self.process_groups["root"] = root_group
self._update_stats()
def _create_example_flow(self):
"""创建示例数据流"""
root_group = self.process_groups["root"]
# 创建示例处理器
generate_processor = Processor(
id="generate-001",
name="GenerateFlowFile",
processor_type=ProcessorType.GENERATE_FLOWFILE
)
generate_processor.add_relationship("success", "成功生成FlowFile")
generate_processor.set_property("File Size", "1KB")
generate_processor.set_property("Batch Size", "1")
generate_processor.set_property("Data Format", "Text")
log_processor = Processor(
id="log-001",
name="LogAttribute",
processor_type=ProcessorType.LOG_ATTRIBUTE
)
log_processor.add_relationship("success", "成功记录属性")
log_processor.set_property("Log Level", "info")
log_processor.set_property("Log Payload", "true")
# 创建连接
connection = Connection(
id="conn-001",
name="GenerateFlowFile to LogAttribute",
source_id="generate-001",
destination_id="log-001",
relationships=["success"]
)
# 添加到根组
root_group.add_processor(generate_processor)
root_group.add_processor(log_processor)
root_group.add_connection(connection)
# 更新全局索引
with self.cluster_lock:
self.processors[generate_processor.id] = generate_processor
self.processors[log_processor.id] = log_processor
self.connections[connection.id] = connection
self._update_stats()
def _update_stats(self):
"""更新统计信息"""
self.stats['total_processors'] = len(self.processors)
self.stats['running_processors'] = len([p for p in self.processors.values() if p.state == ProcessorState.RUNNING])
self.stats['stopped_processors'] = len([p for p in self.processors.values() if p.state == ProcessorState.STOPPED])
self.stats['invalid_processors'] = len([p for p in self.processors.values() if p.state == ProcessorState.INVALID])
self.stats['total_connections'] = len(self.connections)
self.stats['total_process_groups'] = len(self.process_groups)
# 计算队列统计
total_queued = 0
total_queued_size = 0
for connection in self.connections.values():
total_queued += connection.queued_count
total_queued_size += connection.queued_size
self.stats['queued_flow_files'] = total_queued
self.stats['queued_content_size'] = total_queued_size
def create_processor(self, group_id: str, processor_name: str, processor_type: ProcessorType,
properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
"""
创建处理器
Args:
group_id: 处理组ID
processor_name: 处理器名称
processor_type: 处理器类型
properties: 处理器属性
Returns:
Dict[str, Any]: 创建结果
"""
if group_id not in self.process_groups:
return {'status': 'error', 'message': f'Process group {group_id} not found'}
processor_id = f"proc-{uuid.uuid4().hex[:8]}"
processor = Processor(
id=processor_id,
name=processor_name,
processor_type=processor_type,
properties=properties or {}
)
# 根据处理器类型添加默认关系
if processor_type in [ProcessorType.GET_FILE, ProcessorType.GET_HTTP, ProcessorType.GET_KAFKA]:
processor.add_relationship("success", "成功获取数据")
processor.add_relationship("failure", "获取数据失败")
elif processor_type in [ProcessorType.PUT_FILE, ProcessorType.PUT_HDFS, ProcessorType.PUT_KAFKA]:
processor.add_relationship("success", "成功写入数据")
processor.add_relationship("failure", "写入数据失败")
elif processor_type == ProcessorType.ROUTE_ON_ATTRIBUTE:
processor.add_relationship("matched", "属性匹配")
processor.add_relationship("unmatched", "属性不匹配")
elif processor_type == ProcessorType.SPLIT_TEXT:
processor.add_relationship("splits", "分割结果")
processor.add_relationship("original", "原始文件")
processor.add_relationship("failure", "分割失败")
else:
processor.add_relationship("success", "处理成功")
processor.add_relationship("failure", "处理失败")
group = self.process_groups[group_id]
group.add_processor(processor)
with self.cluster_lock:
self.processors[processor_id] = processor
self._update_stats()
return {
'status': 'success',
'processor_id': processor_id,
'processor_name': processor_name,
'processor_type': processor_type.value,
'group_id': group_id,
'message': f'Processor {processor_name} created successfully'
}
def create_connection(self, source_id: str, destination_id: str, relationships: List[str],
name: Optional[str] = None) -> Dict[str, Any]:
"""
创建连接
Args:
source_id: 源处理器ID
destination_id: 目标处理器ID
relationships: 关系列表
name: 连接名称
Returns:
Dict[str, Any]: 创建结果
"""
if source_id not in self.processors:
return {'status': 'error', 'message': f'Source processor {source_id} not found'}
if destination_id not in self.processors:
return {'status': 'error', 'message': f'Destination processor {destination_id} not found'}
source_processor = self.processors[source_id]
destination_processor = self.processors[destination_id]
# 验证关系是否存在
for rel in relationships:
if rel not in source_processor.relationships:
return {'status': 'error', 'message': f'Relationship {rel} not found in source processor'}
connection_id = f"conn-{uuid.uuid4().hex[:8]}"
connection_name = name or f"{source_processor.name} to {destination_processor.name}"
connection = Connection(
id=connection_id,
name=connection_name,
source_id=source_id,
destination_id=destination_id,
relationships=relationships.copy()
)
# 更新处理器连接信息
source_processor.outgoing_connections.append(connection_id)
destination_processor.incoming_connections.append(connection_id)
# 找到处理器所在的组并添加连接
for group in self.process_groups.values():
if source_id in group.processors:
group.add_connection(connection)
break
with self.cluster_lock:
self.connections[connection_id] = connection
self._update_stats()
return {
'status': 'success',
'connection_id': connection_id,
'connection_name': connection_name,
'source_id': source_id,
'destination_id': destination_id,
'relationships': relationships,
'message': f'Connection {connection_name} created successfully'
}
def start_processor(self, processor_id: str) -> Dict[str, Any]:
"""
启动处理器
Args:
processor_id: 处理器ID
Returns:
Dict[str, Any]: 启动结果
"""
if processor_id not in self.processors:
return {'status': 'error', 'message': f'Processor {processor_id} not found'}
processor = self.processors[processor_id]
if processor.state == ProcessorState.RUNNING:
return {'status': 'error', 'message': f'Processor {processor.name} is already running'}
if processor.state == ProcessorState.INVALID:
return {'status': 'error', 'message': f'Processor {processor.name} is invalid and cannot be started'}
processor.state = ProcessorState.RUNNING
with self.cluster_lock:
self._update_stats()
return {
'status': 'success',
'processor_id': processor_id,
'processor_name': processor.name,
'message': f'Processor {processor.name} started successfully'
}
def stop_processor(self, processor_id: str) -> Dict[str, Any]:
"""
停止处理器
Args:
processor_id: 处理器ID
Returns:
Dict[str, Any]: 停止结果
"""
if processor_id not in self.processors:
return {'status': 'error', 'message': f'Processor {processor_id} not found'}
processor = self.processors[processor_id]
if processor.state == ProcessorState.STOPPED:
return {'status': 'error', 'message': f'Processor {processor.name} is already stopped'}
processor.state = ProcessorState.STOPPED
with self.cluster_lock:
self._update_stats()
return {
'status': 'success',
'processor_id': processor_id,
'processor_name': processor.name,
'message': f'Processor {processor.name} stopped successfully'
}
def create_flow_file(self, processor_id: str, content: Union[str, bytes],
attributes: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
"""
创建FlowFile
Args:
processor_id: 处理器ID
content: 内容
attributes: 属性
Returns:
Dict[str, Any]: 创建结果
"""
if processor_id not in self.processors:
return {'status': 'error', 'message': f'Processor {processor_id} not found'}
processor = self.processors[processor_id]
if processor.state != ProcessorState.RUNNING:
return {'status': 'error', 'message': f'Processor {processor.name} is not running'}
# 创建FlowFile
flow_file_uuid = f"ff-{uuid.uuid4().hex}"
content_bytes = content.encode('utf-8') if isinstance(content, str) else content
flow_file = FlowFile(
uuid=flow_file_uuid,
attributes=attributes or {},
content=content_bytes,
size=len(content_bytes)
)
# 设置默认属性
flow_file.put_attribute("filename", f"flowfile-{flow_file_uuid[:8]}")
flow_file.put_attribute("path", "/")
flow_file.put_attribute("uuid", flow_file_uuid)
# 更新处理器统计
processor.flow_files_received += 1
processor.bytes_read += flow_file.size
processor.invocations += 1
return {
'status': 'success',
'flow_file_uuid': flow_file_uuid,
'size': flow_file.size,
'attributes': flow_file.attributes,
'message': f'FlowFile created successfully'
}
def transfer_flow_file(self, processor_id: str, flow_file_uuid: str,
relationship: str) -> Dict[str, Any]:
"""
传输FlowFile
Args:
processor_id: 处理器ID
flow_file_uuid: FlowFile UUID
relationship: 关系名称
Returns:
Dict[str, Any]: 传输结果
"""
if processor_id not in self.processors:
return {'status': 'error', 'message': f'Processor {processor_id} not found'}
processor = self.processors[processor_id]
if relationship not in processor.relationships:
return {'status': 'error', 'message': f'Relationship {relationship} not found'}
# 查找目标连接
target_connections = []
for conn_id in processor.outgoing_connections:
connection = self.connections[conn_id]
if relationship in connection.relationships:
target_connections.append(connection)
if not target_connections:
# 检查是否自动终止
if processor.relationships[relationship].auto_terminate:
processor.flow_files_removed += 1
return {
'status': 'success',
'action': 'auto_terminated',
'message': f'FlowFile auto-terminated on relationship {relationship}'
}
else:
return {'status': 'error', 'message': f'No connection found for relationship {relationship}'}
# 创建模拟FlowFile（实际应该从会话中获取）
flow_file = FlowFile(
uuid=flow_file_uuid,
attributes={'filename': f'flowfile-{flow_file_uuid[:8]}'},
content=b'sample content',
size=14
)
# 传输到所有目标连接
transferred_count = 0
for connection in target_connections:
connection.enqueue(flow_file.clone())
transferred_count += 1
# 更新处理器统计
processor.flow_files_sent += transferred_count
processor.bytes_written += flow_file.size * transferred_count
with self.cluster_lock:
self._update_stats()
return {
'status': 'success',
'flow_file_uuid': flow_file_uuid,
'relationship': relationship,
'transferred_count': transferred_count,
'target_connections': [conn.id for conn in target_connections],
'message': f'FlowFile transferred to {transferred_count} connections'
}
def get_processor_status(self, processor_id: str) -> Dict[str, Any]:
"""
获取处理器状态
Args:
processor_id: 处理器ID
Returns:
Dict[str, Any]: 处理器状态
"""
if processor_id not in self.processors:
return {'status': 'error', 'message': f'Processor {processor_id} not found'}
processor = self.processors[processor_id]
# 计算连接队列信息
incoming_queued = 0
outgoing_queued = 0
for conn_id in processor.incoming_connections:
if conn_id in self.connections:
incoming_queued += self.connections[conn_id].queued_count
for conn_id in processor.outgoing_connections:
if conn_id in self.connections:
outgoing_queued += self.connections[conn_id].queued_count
return {
'status': 'success',
'processor': {
'id': processor.id,
'name': processor.name,
'type': processor.processor_type.value,
'state': processor.state.value,
'properties': processor.properties,
'relationships': {name: rel.name for name, rel in processor.relationships.items()},
'incoming_connections': processor.incoming_connections,
'outgoing_connections': processor.outgoing_connections,
'concurrent_tasks': processor.concurrent_tasks,
'scheduling_period': processor.scheduling_period,
'scheduling_strategy': processor.scheduling_strategy,
'stats': {
'bytes_read': processor.bytes_read,
'bytes_written': processor.bytes_written,
'bytes_transferred': processor.bytes_transferred,
'flow_files_received': processor.flow_files_received,
'flow_files_sent': processor.flow_files_sent,
'flow_files_removed': processor.flow_files_removed,
'invocations': processor.invocations,
'processing_nanos': processor.processing_nanos,
'incoming_queued': incoming_queued,
'outgoing_queued': outgoing_queued
}
}
}
def get_connection_status(self, connection_id: str) -> Dict[str, Any]:
"""
获取连接状态
Args:
connection_id: 连接ID
Returns:
Dict[str, Any]: 连接状态
"""
if connection_id not in self.connections:
return {'status': 'error', 'message': f'Connection {connection_id} not found'}
connection = self.connections[connection_id]
return {
'status': 'success',
'connection': {
'id': connection.id,
'name': connection.name,
'source_id': connection.source_id,
'destination_id': connection.destination_id,
'relationships': connection.relationships,
'connection_type': connection.connection_type.value,
'queue_priority': connection.queue_priority.value,
'back_pressure_object_threshold': connection.back_pressure_object_threshold,
'back_pressure_data_size_threshold': connection.back_pressure_data_size_threshold,
'load_balance_strategy': connection.load_balance_strategy,
'stats': {
'queued_count': connection.queued_count,
'queued_size': connection.queued_size,
'is_back_pressure_enabled': connection.is_back_pressure_enabled()
}
}
}
def list_processors(self, group_id: Optional[str] = None) -> Dict[str, Any]:
"""
列出处理器
Args:
group_id: 处理组ID（可选）
Returns:
Dict[str, Any]: 处理器列表
"""
processors_info = []
if group_id:
if group_id not in self.process_groups:
return {'status': 'error', 'message': f'Process group {group_id} not found'}
group = self.process_groups[group_id]
target_processors = group.processors
else:
target_processors = self.processors
for processor_id, processor in target_processors.items():
processors_info.append({
'id': processor.id,
'name': processor.name,
'type': processor.processor_type.value,
'state': processor.state.value,
'group_id': group_id or 'root',
'concurrent_tasks': processor.concurrent_tasks,
'flow_files_received': processor.flow_files_received,
'flow_files_sent': processor.flow_files_sent
})
return {
'status': 'success',
'processors': processors_info,
'total': len(processors_info)
}
def get_cluster_status(self) -> Dict[str, Any]:
"""
获取集群状态
Returns:
Dict[str, Any]: 集群状态
"""
# 计算5分钟统计（模拟）
total_bytes_read = sum(p.bytes_read for p in self.processors.values())
total_bytes_written = sum(p.bytes_written for p in self.processors.values())
total_flow_files_received = sum(p.flow_files_received for p in self.processors.values())
total_flow_files_sent = sum(p.flow_files_sent for p in self.processors.values())
self.stats['bytes_read_5_min'] = total_bytes_read
self.stats['bytes_written_5_min'] = total_bytes_written
self.stats['bytes_transferred_5_min'] = total_bytes_read + total_bytes_written
self.stats['flow_files_received_5_min'] = total_flow_files_received
self.stats['flow_files_sent_5_min'] = total_flow_files_sent
self.stats['active_threads'] = len([p for p in self.processors.values() if p.state == ProcessorState.RUNNING]) * 2
return {
'cluster_name': self.cluster_name,
'is_running': self.is_running,
'version': self.version,
'java_version': self.java_version,
'cluster_coordinator': self.cluster_coordinator,
'stats': self.stats,
'config': self.config,
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建NiFi集群
nifi = NiFiCluster("production-nifi")
print("=== Apache NiFi数据流处理示例 ===")
# 创建处理器
print("\n=== 创建处理器 ===")
# 创建文件获取处理器
get_file_result = nifi.create_processor(
group_id="root",
processor_name="GetFile-WebLogs",
processor_type=ProcessorType.GET_FILE,
properties={
'Input Directory': '/var/log/web',
'File Filter': '.*\.log$',
'Keep Source File': 'false',
'Minimum File Age': '0 sec',
'Polling Interval': '10 sec',
'Batch Size': '10'
}
)
print(f"GetFile处理器创建结果: {get_file_result}")
# 创建文本替换处理器
replace_text_result = nifi.create_processor(
group_id="root",
processor_name="ReplaceText-Anonymize",
processor_type=ProcessorType.REPLACE_TEXT,
properties={
'Search Value': r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b',
'Replacement Value': 'XXX.XXX.XXX.XXX',
'Character Set': 'UTF-8',
'Maximum Buffer Size': '1 MB',
'Replacement Strategy': 'Regex Replace'
}
)
print(f"ReplaceText处理器创建结果: {replace_text_result}")
# 创建HDFS写入处理器
put_hdfs_result = nifi.create_processor(
group_id="root",
processor_name="PutHDFS-Archive",
processor_type=ProcessorType.PUT_HDFS,
properties={
'Hadoop Configuration Resources': '/etc/hadoop/conf/core-site.xml,/etc/hadoop/conf/hdfs-site.xml',
'Directory': '/data/logs/web/${now():format("yyyy/MM/dd")}',
'Conflict Resolution Strategy': 'replace',
'Block Size': '128 MB',
'Replication': '3',
'Permissions umask': '022'
}
)
print(f"PutHDFS处理器创建结果: {put_hdfs_result}")
if (get_file_result['status'] == 'success' and
replace_text_result['status'] == 'success' and
put_hdfs_result['status'] == 'success'):
get_file_id = get_file_result['processor_id']
replace_text_id = replace_text_result['processor_id']
put_hdfs_id = put_hdfs_result['processor_id']
# 创建连接
print("\n=== 创建连接 ===")
# GetFile -> ReplaceText
conn1_result = nifi.create_connection(
source_id=get_file_id,
destination_id=replace_text_id,
relationships=["success"],
name="GetFile to ReplaceText"
)
print(f"连接1创建结果: {conn1_result}")
# ReplaceText -> PutHDFS
conn2_result = nifi.create_connection(
source_id=replace_text_id,
destination_id=put_hdfs_id,
relationships=["success"],
name="ReplaceText to PutHDFS"
)
print(f"连接2创建结果: {conn2_result}")
# 启动处理器
print("\n=== 启动处理器 ===")
start_results = []
for processor_id in [get_file_id, replace_text_id, put_hdfs_id]:
result = nifi.start_processor(processor_id)
start_results.append(result)
print(f"处理器 {processor_id} 启动结果: {result}")
if all(r['status'] == 'success' for r in start_results):
# 模拟FlowFile处理
print("\n=== 模拟FlowFile处理 ===")
# 创建FlowFile
for i in range(5):
flow_file_result = nifi.create_flow_file(
processor_id=get_file_id,
content=f"192.168.1.{100+i} - - [25/Dec/2023:10:00:{i:02d} +0000] \"GET /api/data HTTP/1.1\" 200 1234\n",
attributes={
'filename': f'access_{i+1}.log',
'path': '/var/log/web/',
'file.size': '78'
}
)
if i < 3: # 只打印前3个结果
print(f"FlowFile {i+1} 创建结果: {flow_file_result}")
if flow_file_result['status'] == 'success':
# 传输FlowFile
transfer_result = nifi.transfer_flow_file(
processor_id=get_file_id,
flow_file_uuid=flow_file_result['flow_file_uuid'],
relationship="success"
)
if i < 3:
print(f"FlowFile {i+1} 传输结果: {transfer_result}")
# 获取处理器状态
print("\n=== 处理器状态 ===")
for processor_id, name in [(get_file_id, "GetFile"), (replace_text_id, "ReplaceText"), (put_hdfs_id, "PutHDFS")]:
status = nifi.get_processor_status(processor_id)
if status['status'] == 'success':
proc_info = status['processor']
stats = proc_info['stats']
print(f"{name} 处理器:")
print(f" 状态: {proc_info['state']}")
print(f" 接收FlowFiles: {stats['flow_files_received']}")
print(f" 发送FlowFiles: {stats['flow_files_sent']}")
print(f" 读取字节: {stats['bytes_read']}")
print(f" 写入字节: {stats['bytes_written']}")
print(f" 传入队列: {stats['incoming_queued']}")
print(f" 传出队列: {stats['outgoing_queued']}")
# 获取连接状态
print("\n=== 连接状态 ===")
if conn1_result['status'] == 'success':
conn_status = nifi.get_connection_status(conn1_result['connection_id'])
if conn_status['status'] == 'success':
conn_info = conn_status['connection']
stats = conn_info['stats']
print(f"连接: {conn_info['name']}")
print(f" 队列数量: {stats['queued_count']}")
print(f" 队列大小: {stats['queued_size']} bytes")
print(f" 背压启用: {stats['is_back_pressure_enabled']}")
# 列出所有处理器
print("\n=== 处理器列表 ===")
processors_list = nifi.list_processors()
if processors_list['status'] == 'success':
print(f"总处理器数: {processors_list['total']}")
for proc in processors_list['processors']:
print(f" - {proc['name']} ({proc['type']}) - 状态: {proc['state']}")
# 获取集群状态
print("\n=== 集群状态 ===")
cluster_status = nifi.get_cluster_status()
print(f"集群名称: {cluster_status['cluster_name']}")
print(f"版本: {cluster_status['version']}")
print(f"运行状态: {cluster_status['is_running']}")
print(f"集群协调器: {cluster_status['cluster_coordinator']}")
print("统计信息:")
for key, value in cluster_status['stats'].items():
print(f" {key}: {value}")
8. 数据传输组件
8.1 Apache Sqoop详解
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime, timedelta
import threading
import time
import random
import json
import uuid
from collections import defaultdict, deque
import re
class SqoopJobType(Enum):
"""Sqoop作业类型"""
IMPORT = "import" # 导入作业
EXPORT = "export" # 导出作业
EVAL = "eval" # 评估作业
LIST_DATABASES = "list-databases" # 列出数据库
LIST_TABLES = "list-tables" # 列出表
CODEGEN = "codegen" # 代码生成
CREATE_HIVE_TABLE = "create-hive-table" # 创建Hive表
HELP = "help" # 帮助
VERSION = "version" # 版本
class SqoopJobStatus(Enum):
"""Sqoop作业状态"""
PENDING = "PENDING" # 等待中
RUNNING = "RUNNING" # 运行中
SUCCEEDED = "SUCCEEDED" # 成功
FAILED = "FAILED" # 失败
KILLED = "KILLED" # 已终止
class DatabaseType(Enum):
"""数据库类型"""
MYSQL = "mysql"
POSTGRESQL = "postgresql"
ORACLE = "oracle"
SQL_SERVER = "sqlserver"
DB2 = "db2"
TERADATA = "teradata"
NETEZZA = "netezza"
HSQLDB = "hsqldb"
DERBY = "derby"
class CompressionCodec(Enum):
"""压缩编解码器"""
NONE = "none"
GZIP = "gzip"
BZIP2 = "bzip2"
LZO = "lzo"
SNAPPY = "snappy"
DEFLATE = "deflate"
class FileFormat(Enum):
"""文件格式"""
TEXT = "text"
SEQUENCE = "sequence"
AVRO = "avro"
PARQUET = "parquet"
ORC = "orc"
@dataclass
class DatabaseConnection:
"""数据库连接"""
connection_id: str
database_type: DatabaseType
host: str
port: int
database_name: str
username: str
password: str = ""
driver_class: str = ""
connection_params: Dict[str, str] = field(default_factory=dict)
max_connections: int = 10
connection_timeout: int = 30 # 秒
created_time: datetime = field(default_factory=datetime.now)
last_used_time: Optional[datetime] = None
is_active: bool = True
def get_jdbc_url(self) -> str:
"""获取JDBC URL"""
if self.database_type == DatabaseType.MYSQL:
return f"jdbc:mysql://{self.host}:{self.port}/{self.database_name}"
elif self.database_type == DatabaseType.POSTGRESQL:
return f"jdbc:postgresql://{self.host}:{self.port}/{self.database_name}"
elif self.database_type == DatabaseType.ORACLE:
return f"jdbc:oracle:thin:@{self.host}:{self.port}:{self.database_name}"
elif self.database_type == DatabaseType.SQL_SERVER:
return f"jdbc:sqlserver://{self.host}:{self.port};databaseName={self.database_name}"
else:
return f"jdbc:{self.database_type.value}://{self.host}:{self.port}/{self.database_name}"
@dataclass
class SqoopImportJob:
"""Sqoop导入作业"""
job_id: str
job_name: str
connection: DatabaseConnection
table_name: str = ""
query: str = ""
target_dir: str = ""
warehouse_dir: str = ""
hive_database: str = "default"
hive_table: str = ""
num_mappers: int = 4
split_by: str = ""
where_clause: str = ""
columns: List[str] = field(default_factory=list)
file_format: FileFormat = FileFormat.TEXT
compression_codec: CompressionCodec = CompressionCodec.NONE
field_delimiter: str = ","
line_delimiter: str = "\n"
null_string: str = "\\N"
null_non_string: str = "\\N"
incremental: bool = False
incremental_column: str = ""
incremental_mode: str = "append" # append, lastmodified
check_column: str = ""
last_value: str = ""
direct: bool = False
as_avrodatafile: bool = False
as_sequencefile: bool = False
as_textfile: bool = True
as_parquetfile: bool = False
hive_import: bool = False
hive_overwrite: bool = False
create_hive_table: bool = False
hive_drop_import_delims: bool = False
map_column_java: Dict[str, str] = field(default_factory=dict)
map_column_hive: Dict[str, str] = field(default_factory=dict)
status: SqoopJobStatus = SqoopJobStatus.PENDING
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
created_time: datetime = field(default_factory=datetime.now)
error_message: str = ""
records_imported: int = 0
bytes_imported: int = 0
map_tasks: int = 0
reduce_tasks: int = 0
@dataclass
class SqoopExportJob:
"""Sqoop导出作业"""
job_id: str
job_name: str
connection: DatabaseConnection
table_name: str
export_dir: str
staging_table: str = ""
clear_staging_table: bool = False
columns: List[str] = field(default_factory=list)
update_key: str = ""
update_mode: str = "updateonly" # updateonly, allowinsert
input_null_string: str = "\\N"
input_null_non_string: str = "\\N"
field_delimiter: str = ","
line_delimiter: str = "\n"
num_mappers: int = 4
batch: bool = False
call: str = ""
direct: bool = False
validate: bool = False
validation_threshold: str = ""
validation_failurehandler: str = ""
status: SqoopJobStatus = SqoopJobStatus.PENDING
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
created_time: datetime = field(default_factory=datetime.now)
error_message: str = ""
records_exported: int = 0
bytes_exported: int = 0
map_tasks: int = 0
reduce_tasks: int = 0
@dataclass
class SqoopJobExecution:
"""Sqoop作业执行"""
execution_id: str
job_id: str
job_type: SqoopJobType
status: SqoopJobStatus
start_time: datetime
end_time: Optional[datetime] = None
duration: Optional[timedelta] = None
map_tasks_total: int = 0
map_tasks_completed: int = 0
map_tasks_failed: int = 0
reduce_tasks_total: int = 0
reduce_tasks_completed: int = 0
reduce_tasks_failed: int = 0
bytes_read: int = 0
bytes_written: int = 0
records_read: int = 0
records_written: int = 0
error_message: str = ""
log_file: str = ""
counters: Dict[str, int] = field(default_factory=dict)
class SqoopServer:
"""
Apache Sqoop数据传输服务器
"""
def __init__(self, server_host: str = "localhost", server_port: int = 12000):
self.server_host = server_host
self.server_port = server_port
self.server_url = f"http://{server_host}:{server_port}/sqoop"
# 数据存储
self.connections = {} # connection_id -> DatabaseConnection
self.import_jobs = {} # job_id -> SqoopImportJob
self.export_jobs = {} # job_id -> SqoopExportJob
self.job_executions = {} # execution_id -> SqoopJobExecution
# 服务器状态
self.is_running = True
self.version = "1.4.7"
self.hadoop_version = "3.3.4"
self.java_version = "1.8.0_281"
# 配置
self.config = {
'sqoop.metastore.client.enable.autoconnect': True,
'sqoop.metastore.client.autoconnect.url': 'jdbc:hsqldb:hsql://localhost:16000/sqoop',
'sqoop.metastore.client.autoconnect.username': 'SA',
'sqoop.metastore.client.autoconnect.password': '',
'sqoop.connection.factories': 'org.apache.sqoop.manager.DefaultManagerFactory',
'sqoop.tool.plugins': '',
'mapreduce.job.user.classpath.first': True,
'mapreduce.map.memory.mb': 1024,
'mapreduce.reduce.memory.mb': 1024
}
# 线程锁
self.connections_lock = threading.Lock()
self.jobs_lock = threading.Lock()
self.executions_lock = threading.Lock()
# 统计信息
self.stats = {
'total_connections': 0,
'active_connections': 0,
'total_import_jobs': 0,
'total_export_jobs': 0,
'running_jobs': 0,
'succeeded_jobs': 0,
'failed_jobs': 0,
'total_records_imported': 0,
'total_records_exported': 0,
'total_bytes_transferred': 0
}
# 初始化示例连接和作业
self._create_example_connections()
self._create_example_jobs()
def _create_example_connections(self):
"""创建示例数据库连接"""
# MySQL连接
mysql_conn = DatabaseConnection(
connection_id="mysql-conn-001",
database_type=DatabaseType.MYSQL,
host="localhost",
port=3306,
database_name="test_db",
username="root",
password="password",
driver_class="com.mysql.cj.jdbc.Driver"
)
# PostgreSQL连接
postgres_conn = DatabaseConnection(
connection_id="postgres-conn-001",
database_type=DatabaseType.POSTGRESQL,
host="localhost",
port=5432,
database_name="test_db",
username="postgres",
password="password",
driver_class="org.postgresql.Driver"
)
with self.connections_lock:
self.connections[mysql_conn.connection_id] = mysql_conn
self.connections[postgres_conn.connection_id] = postgres_conn
self.stats['total_connections'] = len(self.connections)
self.stats['active_connections'] = len([c for c in self.connections.values() if c.is_active])
def _create_example_jobs(self):
"""创建示例作业"""
if not self.connections:
return
mysql_conn = list(self.connections.values())[0]
# 创建示例导入作业
import_job = SqoopImportJob(
job_id=f"import-{uuid.uuid4().hex[:8]}",
job_name="import-users-table",
connection=mysql_conn,
table_name="users",
target_dir="/user/sqoop/import/users",
num_mappers=4,
split_by="id",
file_format=FileFormat.TEXT,
compression_codec=CompressionCodec.GZIP,
hive_import=True,
hive_database="default",
hive_table="users"
)
# 创建示例导出作业
export_job = SqoopExportJob(
job_id=f"export-{uuid.uuid4().hex[:8]}",
job_name="export-processed-data",
connection=mysql_conn,
table_name="processed_data",
export_dir="/user/sqoop/export/processed",
num_mappers=2,
update_mode="allowinsert"
)
with self.jobs_lock:
self.import_jobs[import_job.job_id] = import_job
self.export_jobs[export_job.job_id] = export_job
self.stats['total_import_jobs'] = len(self.import_jobs)
self.stats['total_export_jobs'] = len(self.export_jobs)
def create_connection(self, database_type: DatabaseType, host: str, port: int,
database_name: str, username: str, password: str = "",
connection_params: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
"""
创建数据库连接
Args:
database_type: 数据库类型
host: 主机地址
port: 端口号
database_name: 数据库名称
username: 用户名
password: 密码
connection_params: 连接参数
Returns:
Dict[str, Any]: 创建结果
"""
connection_id = f"{database_type.value}-conn-{uuid.uuid4().hex[:8]}"
# 设置驱动类
driver_class = ""
if database_type == DatabaseType.MYSQL:
driver_class = "com.mysql.cj.jdbc.Driver"
elif database_type == DatabaseType.POSTGRESQL:
driver_class = "org.postgresql.Driver"
elif database_type == DatabaseType.ORACLE:
driver_class = "oracle.jdbc.OracleDriver"
elif database_type == DatabaseType.SQL_SERVER:
driver_class = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
connection = DatabaseConnection(
connection_id=connection_id,
database_type=database_type,
host=host,
port=port,
database_name=database_name,
username=username,
password=password,
driver_class=driver_class,
connection_params=connection_params or {}
)
with self.connections_lock:
self.connections[connection_id] = connection
self.stats['total_connections'] = len(self.connections)
self.stats['active_connections'] = len([c for c in self.connections.values() if c.is_active])
return {
'status': 'success',
'connection_id': connection_id,
'jdbc_url': connection.get_jdbc_url(),
'message': f'Database connection created successfully'
}
def test_connection(self, connection_id: str) -> Dict[str, Any]:
"""
测试数据库连接
Args:
connection_id: 连接ID
Returns:
Dict[str, Any]: 测试结果
"""
if connection_id not in self.connections:
return {'status': 'error', 'message': f'Connection {connection_id} not found'}
connection = self.connections[connection_id]
# 模拟连接测试
test_success = random.random() > 0.1 # 90%成功率
if test_success:
connection.last_used_time = datetime.now()
return {
'status': 'success',
'connection_id': connection_id,
'jdbc_url': connection.get_jdbc_url(),
'message': 'Connection test successful'
}
else:
return {
'status': 'error',
'connection_id': connection_id,
'message': 'Connection test failed: Unable to connect to database'
}
def list_databases(self, connection_id: str) -> Dict[str, Any]:
"""
列出数据库
Args:
connection_id: 连接ID
Returns:
Dict[str, Any]: 数据库列表
"""
if connection_id not in self.connections:
return {'status': 'error', 'message': f'Connection {connection_id} not found'}
connection = self.connections[connection_id]
# 模拟数据库列表
databases = [
connection.database_name,
'information_schema',
'mysql' if connection.database_type == DatabaseType.MYSQL else 'postgres',
'performance_schema' if connection.database_type == DatabaseType.MYSQL else 'template0',
'sys' if connection.database_type == DatabaseType.MYSQL else 'template1'
]
return {
'status': 'success',
'connection_id': connection_id,
'databases': databases,
'count': len(databases)
}
def list_tables(self, connection_id: str, database_name: Optional[str] = None) -> Dict[str, Any]:
"""
列出表
Args:
connection_id: 连接ID
database_name: 数据库名称
Returns:
Dict[str, Any]: 表列表
"""
if connection_id not in self.connections:
return {'status': 'error', 'message': f'Connection {connection_id} not found'}
connection = self.connections[connection_id]
db_name = database_name or connection.database_name
# 模拟表列表
tables = [
'users',
'orders',
'products',
'categories',
'order_items',
'user_profiles',
'audit_log',
'system_config'
]
return {
'status': 'success',
'connection_id': connection_id,
'database': db_name,
'tables': tables,
'count': len(tables)
}
def create_import_job(self, job_name: str, connection_id: str,
table_name: str = "", query: str = "",
target_dir: str = "", **kwargs) -> Dict[str, Any]:
"""
创建导入作业
Args:
job_name: 作业名称
connection_id: 连接ID
table_name: 表名
query: 查询语句
target_dir: 目标目录
**kwargs: 其他参数
Returns:
Dict[str, Any]: 创建结果
"""
if connection_id not in self.connections:
return {'status': 'error', 'message': f'Connection {connection_id} not found'}
if not table_name and not query:
return {'status': 'error', 'message': 'Either table_name or query must be specified'}
connection = self.connections[connection_id]
job_id = f"import-{uuid.uuid4().hex[:8]}"
import_job = SqoopImportJob(
job_id=job_id,
job_name=job_name,
connection=connection,
table_name=table_name,
query=query,
target_dir=target_dir or f"/user/sqoop/import/{table_name or 'query_result'}",
num_mappers=kwargs.get('num_mappers', 4),
split_by=kwargs.get('split_by', ''),
where_clause=kwargs.get('where_clause', ''),
columns=kwargs.get('columns', []),
file_format=FileFormat(kwargs.get('file_format', 'text')),
compression_codec=CompressionCodec(kwargs.get('compression_codec', 'none')),
field_delimiter=kwargs.get('field_delimiter', ','),
hive_import=kwargs.get('hive_import', False),
hive_database=kwargs.get('hive_database', 'default'),
hive_table=kwargs.get('hive_table', table_name),
incremental=kwargs.get('incremental', False),
incremental_column=kwargs.get('incremental_column', ''),
incremental_mode=kwargs.get('incremental_mode', 'append')
)
with self.jobs_lock:
self.import_jobs[job_id] = import_job
self.stats['total_import_jobs'] = len(self.import_jobs)
return {
'status': 'success',
'job_id': job_id,
'job_name': job_name,
'message': f'Import job created successfully'
}
def create_export_job(self, job_name: str, connection_id: str,
table_name: str, export_dir: str, **kwargs) -> Dict[str, Any]:
"""
创建导出作业
Args:
job_name: 作业名称
connection_id: 连接ID
table_name: 表名
export_dir: 导出目录
**kwargs: 其他参数
Returns:
Dict[str, Any]: 创建结果
"""
if connection_id not in self.connections:
return {'status': 'error', 'message': f'Connection {connection_id} not found'}
connection = self.connections[connection_id]
job_id = f"export-{uuid.uuid4().hex[:8]}"
export_job = SqoopExportJob(
job_id=job_id,
job_name=job_name,
connection=connection,
table_name=table_name,
export_dir=export_dir,
staging_table=kwargs.get('staging_table', ''),
columns=kwargs.get('columns', []),
update_key=kwargs.get('update_key', ''),
update_mode=kwargs.get('update_mode', 'updateonly'),
field_delimiter=kwargs.get('field_delimiter', ','),
num_mappers=kwargs.get('num_mappers', 4),
batch=kwargs.get('batch', False),
direct=kwargs.get('direct', False)
)
with self.jobs_lock:
self.export_jobs[job_id] = export_job
self.stats['total_export_jobs'] = len(self.export_jobs)
return {
'status': 'success',
'job_id': job_id,
'job_name': job_name,
'message': f'Export job created successfully'
}
def execute_import_job(self, job_id: str) -> Dict[str, Any]:
"""
执行导入作业
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 执行结果
"""
if job_id not in self.import_jobs:
return {'status': 'error', 'message': f'Import job {job_id} not found'}
import_job = self.import_jobs[job_id]
if import_job.status == SqoopJobStatus.RUNNING:
return {'status': 'error', 'message': f'Import job {job_id} is already running'}
# 创建执行记录
execution_id = f"exec-{uuid.uuid4().hex[:8]}"
execution = SqoopJobExecution(
execution_id=execution_id,
job_id=job_id,
job_type=SqoopJobType.IMPORT,
status=SqoopJobStatus.RUNNING,
start_time=datetime.now(),
map_tasks_total=import_job.num_mappers
)
with self.jobs_lock:
import_job.status = SqoopJobStatus.RUNNING
import_job.start_time = datetime.now()
self.stats['running_jobs'] += 1
with self.executions_lock:
self.job_executions[execution_id] = execution
# 模拟异步执行
def execute_async():
time.sleep(random.uniform(2, 8)) # 模拟执行时间
# 模拟执行结果（85%成功率）
success = random.random() > 0.15
with self.jobs_lock:
if success:
import_job.status = SqoopJobStatus.SUCCEEDED
import_job.records_imported = random.randint(1000, 100000)
import_job.bytes_imported = import_job.records_imported * random.randint(50, 200)
execution.status = SqoopJobStatus.SUCCEEDED
execution.records_read = import_job.records_imported
execution.bytes_read = import_job.bytes_imported
execution.map_tasks_completed = import_job.num_mappers
self.stats['succeeded_jobs'] += 1
self.stats['total_records_imported'] += import_job.records_imported
self.stats['total_bytes_transferred'] += import_job.bytes_imported
else:
import_job.status = SqoopJobStatus.FAILED
import_job.error_message = "Import failed: Connection timeout"
execution.status = SqoopJobStatus.FAILED
execution.error_message = import_job.error_message
execution.map_tasks_failed = import_job.num_mappers
self.stats['failed_jobs'] += 1
import_job.end_time = datetime.now()
execution.end_time = datetime.now()
execution.duration = execution.end_time - execution.start_time
self.stats['running_jobs'] -= 1
threading.Thread(target=execute_async, daemon=True).start()
return {
'status': 'success',
'job_id': job_id,
'execution_id': execution_id,
'message': f'Import job started successfully'
}
def execute_export_job(self, job_id: str) -> Dict[str, Any]:
"""
执行导出作业
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 执行结果
"""
if job_id not in self.export_jobs:
return {'status': 'error', 'message': f'Export job {job_id} not found'}
export_job = self.export_jobs[job_id]
if export_job.status == SqoopJobStatus.RUNNING:
return {'status': 'error', 'message': f'Export job {job_id} is already running'}
# 创建执行记录
execution_id = f"exec-{uuid.uuid4().hex[:8]}"
execution = SqoopJobExecution(
execution_id=execution_id,
job_id=job_id,
job_type=SqoopJobType.EXPORT,
status=SqoopJobStatus.RUNNING,
start_time=datetime.now(),
map_tasks_total=export_job.num_mappers
)
with self.jobs_lock:
export_job.status = SqoopJobStatus.RUNNING
export_job.start_time = datetime.now()
self.stats['running_jobs'] += 1
with self.executions_lock:
self.job_executions[execution_id] = execution
# 模拟异步执行
def execute_async():
time.sleep(random.uniform(2, 6)) # 模拟执行时间
# 模拟执行结果（80%成功率）
success = random.random() > 0.2
with self.jobs_lock:
if success:
export_job.status = SqoopJobStatus.SUCCEEDED
export_job.records_exported = random.randint(500, 50000)
export_job.bytes_exported = export_job.records_exported * random.randint(40, 150)
execution.status = SqoopJobStatus.SUCCEEDED
execution.records_written = export_job.records_exported
execution.bytes_written = export_job.bytes_exported
execution.map_tasks_completed = export_job.num_mappers
self.stats['succeeded_jobs'] += 1
self.stats['total_records_exported'] += export_job.records_exported
self.stats['total_bytes_transferred'] += export_job.bytes_exported
else:
export_job.status = SqoopJobStatus.FAILED
export_job.error_message = "Export failed: Table does not exist"
execution.status = SqoopJobStatus.FAILED
execution.error_message = export_job.error_message
execution.map_tasks_failed = export_job.num_mappers
self.stats['failed_jobs'] += 1
export_job.end_time = datetime.now()
execution.end_time = datetime.now()
execution.duration = execution.end_time - execution.start_time
self.stats['running_jobs'] -= 1
threading.Thread(target=execute_async, daemon=True).start()
return {
'status': 'success',
'job_id': job_id,
'execution_id': execution_id,
'message': f'Export job started successfully'
}
def get_job_status(self, job_id: str) -> Dict[str, Any]:
"""
获取作业状态
Args:
job_id: 作业ID
Returns:
Dict[str, Any]: 作业状态
"""
# 检查导入作业
if job_id in self.import_jobs:
job = self.import_jobs[job_id]
return {
'status': 'success',
'job': {
'job_id': job.job_id,
'job_name': job.job_name,
'job_type': 'import',
'status': job.status.value,
'table_name': job.table_name,
'target_dir': job.target_dir,
'num_mappers': job.num_mappers,
'start_time': job.start_time.isoformat() if job.start_time else None,
'end_time': job.end_time.isoformat() if job.end_time else None,
'created_time': job.created_time.isoformat(),
'records_imported': job.records_imported,
'bytes_imported': job.bytes_imported,
'error_message': job.error_message,
'connection_id': job.connection.connection_id,
'database_type': job.connection.database_type.value
}
}
# 检查导出作业
if job_id in self.export_jobs:
job = self.export_jobs[job_id]
return {
'status': 'success',
'job': {
'job_id': job.job_id,
'job_name': job.job_name,
'job_type': 'export',
'status': job.status.value,
'table_name': job.table_name,
'export_dir': job.export_dir,
'num_mappers': job.num_mappers,
'start_time': job.start_time.isoformat() if job.start_time else None,
'end_time': job.end_time.isoformat() if job.end_time else None,
'created_time': job.created_time.isoformat(),
'records_exported': job.records_exported,
'bytes_exported': job.bytes_exported,
'error_message': job.error_message,
'connection_id': job.connection.connection_id,
'database_type': job.connection.database_type.value
}
}
return {'status': 'error', 'message': f'Job {job_id} not found'}
def get_execution_status(self, execution_id: str) -> Dict[str, Any]:
"""
获取执行状态
Args:
execution_id: 执行ID
Returns:
Dict[str, Any]: 执行状态
"""
if execution_id not in self.job_executions:
return {'status': 'error', 'message': f'Execution {execution_id} not found'}
execution = self.job_executions[execution_id]
return {
'status': 'success',
'execution': {
'execution_id': execution.execution_id,
'job_id': execution.job_id,
'job_type': execution.job_type.value,
'status': execution.status.value,
'start_time': execution.start_time.isoformat(),
'end_time': execution.end_time.isoformat() if execution.end_time else None,
'duration': str(execution.duration) if execution.duration else None,
'map_tasks_total': execution.map_tasks_total,
'map_tasks_completed': execution.map_tasks_completed,
'map_tasks_failed': execution.map_tasks_failed,
'records_read': execution.records_read,
'records_written': execution.records_written,
'bytes_read': execution.bytes_read,
'bytes_written': execution.bytes_written,
'error_message': execution.error_message,
'counters': execution.counters
}
}
def list_jobs(self, job_type: Optional[str] = None, status: Optional[str] = None,
limit: int = 50) -> Dict[str, Any]:
"""
列出作业
Args:
job_type: 作业类型过滤 (import, export)
status: 状态过滤
limit: 限制数量
Returns:
Dict[str, Any]: 作业列表
"""
jobs_info = []
count = 0
# 添加导入作业
if not job_type or job_type == 'import':
for job_id, job in self.import_jobs.items():
if count >= limit:
break
if status and job.status.value != status:
continue
jobs_info.append({
'job_id': job.job_id,
'job_name': job.job_name,
'job_type': 'import',
'status': job.status.value,
'table_name': job.table_name,
'target_dir': job.target_dir,
'created_time': job.created_time.isoformat(),
'start_time': job.start_time.isoformat() if job.start_time else None,
'records_imported': job.records_imported,
'connection_id': job.connection.connection_id
})
count += 1
# 添加导出作业
if not job_type or job_type == 'export':
for job_id, job in self.export_jobs.items():
if count >= limit:
break
if status and job.status.value != status:
continue
jobs_info.append({
'job_id': job.job_id,
'job_name': job.job_name,
'job_type': 'export',
'status': job.status.value,
'table_name': job.table_name,
'export_dir': job.export_dir,
'created_time': job.created_time.isoformat(),
'start_time': job.start_time.isoformat() if job.start_time else None,
'records_exported': job.records_exported,
'connection_id': job.connection.connection_id
})
count += 1
return {
'status': 'success',
'jobs': jobs_info,
'total': len(jobs_info),
'job_type_filter': job_type,
'status_filter': status
}
def list_connections(self) -> Dict[str, Any]:
"""
列出数据库连接
Returns:
Dict[str, Any]: 连接列表
"""
connections_info = []
for conn_id, conn in self.connections.items():
connections_info.append({
'connection_id': conn.connection_id,
'database_type': conn.database_type.value,
'host': conn.host,
'port': conn.port,
'database_name': conn.database_name,
'username': conn.username,
'jdbc_url': conn.get_jdbc_url(),
'is_active': conn.is_active,
'created_time': conn.created_time.isoformat(),
'last_used_time': conn.last_used_time.isoformat() if conn.last_used_time else None,
'max_connections': conn.max_connections
})
return {
'status': 'success',
'connections': connections_info,
'total': len(connections_info)
}
def get_server_status(self) -> Dict[str, Any]:
"""
获取服务器状态
Returns:
Dict[str, Any]: 服务器状态
"""
return {
'server_url': self.server_url,
'is_running': self.is_running,
'version': self.version,
'hadoop_version': self.hadoop_version,
'java_version': self.java_version,
'stats': self.stats,
'config': self.config,
'timestamp': datetime.now().isoformat()
}
# 使用示例
if __name__ == "__main__":
# 创建Sqoop服务器
sqoop = SqoopServer("localhost", 12000)
print("=== Apache Sqoop数据传输示例 ===")
# 创建数据库连接
print("\n=== 创建数据库连接 ===")
mysql_conn_result = sqoop.create_connection(
database_type=DatabaseType.MYSQL,
host="localhost",
port=3306,
database_name="ecommerce",
username="sqoop_user",
password="password123"
)
print(f"MySQL连接创建结果: {mysql_conn_result}")
if mysql_conn_result['status'] == 'success':
mysql_conn_id = mysql_conn_result['connection_id']
# 测试连接
print("\n=== 测试数据库连接 ===")
test_result = sqoop.test_connection(mysql_conn_id)
print(f"连接测试结果: {test_result}")
# 列出数据库
print("\n=== 列出数据库 ===")
databases = sqoop.list_databases(mysql_conn_id)
if databases['status'] == 'success':
print(f"数据库列表 ({databases['count']}个):")
for db in databases['databases']:
print(f" - {db}")
# 列出表
print("\n=== 列出表 ===")
tables = sqoop.list_tables(mysql_conn_id)
if tables['status'] == 'success':
print(f"表列表 ({tables['count']}个):")
for table in tables['tables']:
print(f" - {table}")
# 创建导入作业
print("\n=== 创建导入作业 ===")
import_job_result = sqoop.create_import_job(
job_name="import-users-data",
connection_id=mysql_conn_id,
table_name="users",
target_dir="/user/sqoop/import/users",
num_mappers=4,
split_by="user_id",
file_format="text",
compression_codec="gzip",
hive_import=True,
hive_database="warehouse",
hive_table="users"
)
print(f"导入作业创建结果: {import_job_result}")
if import_job_result['status'] == 'success':
import_job_id = import_job_result['job_id']
# 执行导入作业
print("\n=== 执行导入作业 ===")
import_exec_result = sqoop.execute_import_job(import_job_id)
print(f"导入作业执行结果: {import_exec_result}")
if import_exec_result['status'] == 'success':
execution_id = import_exec_result['execution_id']
# 等待一段时间
time.sleep(3)
# 获取作业状态
print("\n=== 导入作业状态 ===")
job_status = sqoop.get_job_status(import_job_id)
if job_status['status'] == 'success':
job = job_status['job']
print(f"作业ID: {job['job_id']}")
print(f"作业名称: {job['job_name']}")
print(f"状态: {job['status']}")
print(f"表名: {job['table_name']}")
print(f"目标目录: {job['target_dir']}")
print(f"导入记录数: {job['records_imported']}")
print(f"导入字节数: {job['bytes_imported']}")
# 获取执行状态
print("\n=== 执行状态详情 ===")
exec_status = sqoop.get_execution_status(execution_id)
if exec_status['status'] == 'success':
execution = exec_status['execution']
print(f"执行ID: {execution['execution_id']}")
print(f"作业类型: {execution['job_type']}")
print(f"状态: {execution['status']}")
print(f"开始时间: {execution['start_time']}")
print(f"结束时间: {execution['end_time']}")
print(f"持续时间: {execution['duration']}")
print(f"Map任务: {execution['map_tasks_completed']}/{execution['map_tasks_total']}")
print(f"读取记录数: {execution['records_read']}")
print(f"读取字节数: {execution['bytes_read']}")
# 创建导出作业
print("\n=== 创建导出作业 ===")
export_job_result = sqoop.create_export_job(
job_name="export-processed-orders",
connection_id=mysql_conn_id,
table_name="processed_orders",
export_dir="/user/hive/warehouse/processed_orders",
num_mappers=2,
update_mode="allowinsert",
field_delimiter="\t"
)
print(f"导出作业创建结果: {export_job_result}")
if export_job_result['status'] == 'success':
export_job_id = export_job_result['job_id']
# 执行导出作业
print("\n=== 执行导出作业 ===")
export_exec_result = sqoop.execute_export_job(export_job_id)
print(f"导出作业执行结果: {export_exec_result}")
# 等待一段时间
time.sleep(2)
# 获取导出作业状态
print("\n=== 导出作业状态 ===")
export_status = sqoop.get_job_status(export_job_id)
if export_status['status'] == 'success':
job = export_status['job']
print(f"作业ID: {job['job_id']}")
print(f"作业名称: {job['job_name']}")
print(f"状态: {job['status']}")
print(f"表名: {job['table_name']}")
print(f"导出目录: {job['export_dir']}")
print(f"导出记录数: {job['records_exported']}")
print(f"导出字节数: {job['bytes_exported']}")
# 列出所有作业
print("\n=== 作业列表 ===")
all_jobs = sqoop.list_jobs(limit=20)
if all_jobs['status'] == 'success':
print(f"总作业数: {all_jobs['total']}")
for job in all_jobs['jobs']:
print(f" - {job['job_id']}: {job['job_name']} ({job['job_type']}) - {job['status']}")
# 列出所有连接
print("\n=== 连接列表 ===")
all_connections = sqoop.list_connections()
if all_connections['status'] == 'success':
print(f"总连接数: {all_connections['total']}")
for conn in all_connections['connections']:
print(f" - {conn['connection_id']}: {conn['database_type']} @ {conn['host']}:{conn['port']}/{conn['database_name']}")
# 获取服务器状态
print("\n=== 服务器状态 ===")
status = sqoop.get_server_status()
print(f"服务器URL: {status['server_url']}")
print(f"运行状态: {status['is_running']}")
print(f"版本: {status['version']}")
print(f"Hadoop版本: {status['hadoop_version']}")
print("统计信息:")
for key, value in status['stats'].items():
print(f" {key}: {value}")
